View Javadoc

    1   /*
    2    * Licensed to the Apache Software Foundation (ASF) under one or more
    3    * contributor license agreements.  See the NOTICE file distributed with
    4    * this work for additional information regarding copyright ownership.
    5    * The ASF licenses this file to You under the Apache License, Version 2.0
    6    * (the "License"); you may not use this file except in compliance with
    7    * the License.  You may obtain a copy of the License at
    8    *
    9    *      http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   */
    17
    18  package org.apache.log4j.varia;
    19
    20  import org.apache.log4j.Level;
    21  import org.apache.log4j.Logger;
    22  import org.apache.log4j.helpers.Constants;
    23  import org.apache.log4j.plugins.Receiver;
    24  import org.apache.log4j.rule.ExpressionRule;
    25  import org.apache.log4j.rule.Rule;
    26  import org.apache.log4j.spi.LocationInfo;
    27  import org.apache.log4j.spi.LoggingEvent;
    28  import org.apache.log4j.spi.ThrowableInformation;
    29
    30  import java.io.*;
    31  import java.net.MalformedURLException;
    32  import java.net.URL;
    33  import java.text.SimpleDateFormat;
    34  import java.util.*;
    35  import java.util.regex.MatchResult;
    36  import java.util.regex.Matcher;
    37  import java.util.regex.Pattern;
    38  import java.util.regex.PatternSyntaxException;
    39
    40  /**
    41   * LogFilePatternReceiver can parse and tail log files, converting entries into
    42   * LoggingEvents.  If the file doesn't exist when the receiver is initialized, the
    43   * receiver will look for the file once every 10 seconds.
    44   * <p>
    45   * This receiver relies on java.util.regex features to perform the parsing of text in the
    46   * log file, however the only regular expression field explicitly supported is
    47   * a glob-style wildcard used to ignore fields in the log file if needed.  All other
    48   * fields are parsed by using the supplied keywords.
    49   * <p>
    50   * <b>Features:</b><br>
    51   * - specify the URL of the log file to be processed<br>
    52   * - specify the timestamp format in the file (if one exists, using patterns from {@link java.text.SimpleDateFormat})<br>
    53   * - specify the pattern (logFormat) used in the log file using keywords, a wildcard character (*) and fixed text<br>
    54   * - 'tail' the file (allows the contents of the file to be continually read and new events processed)<br>
    55   * - supports the parsing of multi-line messages and exceptions
    56   * - 'hostname' property set to URL host (or 'file' if not available)
    57   * - 'application' property set to URL path (or value of fileURL if not available)
    58   * - 'group' property can be set to associate multiple log file receivers
    59   * <p>
    60   * <b>Keywords:</b><br>
    61   * TIMESTAMP<br>
    62   * LOGGER<br>
    63   * LEVEL<br>
    64   * THREAD<br>
    65   * CLASS<br>
    66   * FILE<br>
    67   * LINE<br>
    68   * METHOD<br>
    69   * RELATIVETIME<br>
    70   * MESSAGE<br>
    71   * NDC<br>
    72   * PROP(key)<br>
    73   * (NL)<br>
    74   * <p>
    75   * (NL) represents a new line embedded in the log format, supporting log formats whose fields span multiple lines
    76   * <p>
    77   * Use a * to ignore portions of the log format that should be ignored
    78   * <p>
    79   * Example:<br>
    80   * If your file's patternlayout is this:<br>
    81   * <b>%d %-5p [%t] %C{2} (%F:%L) - %m%n</b>
    82   * <p>
    83   * specify this as the log format:<br>
    84   * <b>TIMESTAMP LEVEL [THREAD] CLASS (FILE:LINE) - MESSAGE</b>
    85   * <p>
    86   * To define a PROPERTY field, use PROP(key)
    87   * <p>
    88   * Example:<br>
    89   * If you used the RELATIVETIME pattern layout character in the file,
    90   * you can use PROP(RELATIVETIME) in the logFormat definition to assign
    91   * the RELATIVETIME field as a property on the event.
    92   * <p>
    93   * If your file's patternlayout is this:<br>
    94   * <b>%r [%t] %-5p %c %x - %m%n</b>
    95   * <p>
    96   * specify this as the log format:<br>
    97   * <b>PROP(RELATIVETIME) [THREAD] LEVEL LOGGER * - MESSAGE</b>
    98   * <p>
    99   * Note the * - it can be used to ignore a single word or sequence of words in the log file
    100  * (in order for the wildcard to ignore a sequence of words, the text being ignored must be
    101  * followed by some delimiter, like '-' or '[') - ndc is being ignored in the following example.
    102  * <p>
    103  * Assign a filterExpression in order to only process events which match a filter.
    104  * If a filterExpression is not assigned, all events are processed.
    105  * <p>
    106  * <b>Limitations:</b><br>
    107  * - no support for the single-line version of throwable supported by patternlayout<br>
    108  * (this version of throwable will be included as the last line of the message)<br>
    109  * - the relativetime patternLayout character must be set as a property: PROP(RELATIVETIME)<br>
    110  * - messages should appear as the last field of the logFormat because the variability in message content<br>
    111  * - exceptions are converted if the exception stack trace (other than the first line of the exception)<br>
    112  * is stored in the log file with a tab followed by the word 'at' as the first characters in the line<br>
    113  * - tailing may fail if the file rolls over.
    114  * <p>
    115  * <b>Example receiver configuration settings</b> (add these as params, specifying a LogFilePatternReceiver 'plugin'):<br>
    116  * param: "timestampFormat" value="yyyy-MM-d HH:mm:ss,SSS"<br>
    117  * param: "logFormat" value="PROP(RELATIVETIME) [THREAD] LEVEL LOGGER * - MESSAGE"<br>
    118  * param: "fileURL" value="file:///c:/events.log"<br>
    119  * param: "tailing" value="true"
    120  * <p>
    121  * This configuration will be able to process these sample events:<br>
    122  * 710    [       Thread-0] DEBUG                   first.logger first - &lt;test&gt;   &lt;test2&gt;something here&lt;/test2&gt;   &lt;test3 blah=something/&gt;   &lt;test4&gt;       &lt;test5&gt;something else&lt;/test5&gt;   &lt;/test4&gt;&lt;/test&gt;<br>
    123  * 880    [       Thread-2] DEBUG                   first.logger third - &lt;test&gt;   &lt;test2&gt;something here&lt;/test2&gt;   &lt;test3 blah=something/&gt;   &lt;test4&gt;       &lt;test5&gt;something else&lt;/test5&gt;   &lt;/test4&gt;&lt;/test&gt;<br>
    124  * 880    [       Thread-0] INFO                    first.logger first - infomsg-0<br>
    125  * java.lang.Exception: someexception-first<br>
    126  * at Generator2.run(Generator2.java:102)<br>
    127  *
    128  * @author Scott Deboy
    129  */
    130 public class LogFilePatternReceiver extends Receiver {
    131     private final List<String> keywords = new ArrayList<>();
    132
    133     private static final String PROP_START = "PROP(";
    134     private static final String PROP_END = ")";
    135
    136     private static final String LOGGER = "LOGGER";
    137     private static final String MESSAGE = "MESSAGE";
    138     private static final String TIMESTAMP = "TIMESTAMP";
    139     private static final String NDC = "NDC";
    140     private static final String LEVEL = "LEVEL";
    141     private static final String THREAD = "THREAD";
    142     private static final String CLASS = "CLASS";
    143     private static final String FILE = "FILE";
    144     private static final String LINE = "LINE";
    145     private static final String METHOD = "METHOD";
    146     private static final String NEWLINE = "(NL)";
    147
    148     private static final String DEFAULT_HOST = "file";
    149
    150     //all lines other than first line of exception begin with tab followed by 'at' followed by text
    151     private static final String EXCEPTION_PATTERN = "^\\s+at.*";
    152     private static final String REGEXP_DEFAULT_WILDCARD = ".*?";
    153     private static final String REGEXP_GREEDY_WILDCARD = ".*";
    154     private static final String PATTERN_WILDCARD = "*";
    155     //pull in optional leading and trailing spaces
    156     private static final String NOSPACE_GROUP = "(\\s*?\\S*?\\s*?)";
    157     private static final String DEFAULT_GROUP = "(" + REGEXP_DEFAULT_WILDCARD + ")";
    158     private static final String GREEDY_GROUP = "(" + REGEXP_GREEDY_WILDCARD + ")";
    159     private static final String MULTIPLE_SPACES_REGEXP = "[ ]+";
    160     private static final String NEWLINE_REGEXP = "\n";
    161     private final String newLine = System.getProperty("line.separator");
    162
    163     private final String[] emptyException = new String[]{""};
    164
    165     private SimpleDateFormat dateFormat;
    166     private String timestampFormat;
    167     private String logFormat;
    168     private String customLevelDefinitions;
    169     private String fileURL;
    170     private String host;
    171     private String path;
    172     private boolean tailing;
    173     private String filterExpression;
    174     private long waitMillis = 2000; //default 2 seconds
    175     private String group;
    176
    177     private static final String VALID_DATEFORMAT_CHARS = "GyYMwWDdFEuaHkKhmsSzZX";
    178     private static final String VALID_DATEFORMAT_CHAR_PATTERN = "[" + VALID_DATEFORMAT_CHARS + "]";
    179
    180     private Rule expressionRule;
    181
    182     private Map currentMap;
    183     private List<String> additionalLines;
    184     private List<String> matchingKeywords;
    185
    186     private String regexp;
    187     private Reader reader;
    188     private Pattern regexpPattern;
    189     private Pattern exceptionPattern;
    190     private String timestampPatternText;
    191
    192     private boolean useCurrentThread;
    193     public static final int MISSING_FILE_RETRY_MILLIS = 10000;
    194     private boolean appendNonMatches;
    195     private final Map<String, Level> customLevelDefinitionMap = new HashMap<>();
    196
    197     //default to one line - this number is incremented for each (NL) found in the logFormat
    198     private int lineCount = 1;
    199
    200     public LogFilePatternReceiver() {
    201         keywords.add(TIMESTAMP);
    202         keywords.add(LOGGER);
    203         keywords.add(LEVEL);
    204         keywords.add(THREAD);
    205         keywords.add(CLASS);
    206         keywords.add(FILE);
    207         keywords.add(LINE);
    208         keywords.add(METHOD);
    209         keywords.add(MESSAGE);
    210         keywords.add(NDC);
    211         try {
    212             exceptionPattern = Pattern.compile(EXCEPTION_PATTERN);
    213         } catch (PatternSyntaxException pse) {
    214             //shouldn't happen
    215         }
    216     }
    217
    218     /**
    219      * Accessor
    220      *
    221      * @return file URL
    222      */
    223     public String getFileURL() {
    224         return fileURL;
    225     }
    226
    227     /**
    228      * Mutator
    229      *
    230      * @param fileURL
    231      */
    232     public void setFileURL(String fileURL) {
    233         this.fileURL = fileURL;
    234     }
    235
    236     /**
    237      * If the log file contains non-log4j level strings, they can be mapped to log4j levels using the format (android example):
    238      * V=TRACE,D=DEBUG,I=INFO,W=WARN,E=ERROR,F=FATAL,S=OFF
    239      *
    240      * @param customLevelDefinitions the level definition string
    241      */
    242     public void setCustomLevelDefinitions(String customLevelDefinitions) {
    243         this.customLevelDefinitions = customLevelDefinitions;
    244     }
    245
    246     public String getCustomLevelDefinitions() {
    247         return customLevelDefinitions;
    248     }
    249
    250     /**
    251      * Accessor
    252      *
    253      * @return append non matches
    254      */
    255     public boolean isAppendNonMatches() {
    256         return appendNonMatches;
    257     }
    258
    259     /**
    260      * Mutator
    261      *
    262      * @param appendNonMatches
    263      */
    264     public void setAppendNonMatches(boolean appendNonMatches) {
    265         this.appendNonMatches = appendNonMatches;
    266     }
    267
    268     /**
    269      * Accessor
    270      *
    271      * @return filter expression
    272      */
    273     public String getFilterExpression() {
    274         return filterExpression;
    275     }
    276
    277     /**
    278      * Mutator
    279      *
    280      * @param filterExpression
    281      */
    282     public void setFilterExpression(String filterExpression) {
    283         this.filterExpression = filterExpression;
    284     }
    285
    286     /**
    287      * Accessor
    288      *
    289      * @return tailing
    290      */
    291     public boolean isTailing() {
    292         return tailing;
    293     }
    294
    295     /**
    296      * Mutator
    297      *
    298      * @param tailing
    299      */
    300     public void setTailing(boolean tailing) {
    301         this.tailing = tailing;
    302     }
    303
    304     /**
    305      * When true, this property uses the current Thread to perform the import,
    306      * otherwise when false (the default), a new Thread is created and started to manage
    307      * the import.
    308      *
    309      * @return
    310      */
    311     public final boolean isUseCurrentThread() {
    312         return useCurrentThread;
    313     }
    314
    315     /**
    316      * Sets whether the current Thread or a new Thread is created to perform the import,
    317      * the default being false (new Thread created).
    318      *
    319      * @param useCurrentThread
    320      */
    321     public final void setUseCurrentThread(boolean useCurrentThread) {
    322         this.useCurrentThread = useCurrentThread;
    323     }
    324
    325     /**
    326      * Accessor
    327      *
    328      * @return log format
    329      */
    330     public String getLogFormat() {
    331         return logFormat;
    332     }
    333
    334     /**
    335      * Mutator
    336      *
    337      * @param logFormat the format
    338      */
    339     public void setLogFormat(String logFormat) {
    340         this.logFormat = logFormat;
    341     }
    342
    343     /**
    344      * Mutator
    345      */
    346     public void setGroup(String group) {
    347         this.group = group;
    348     }
    349
    350
    351     /**
    352      * Accessor
    353      *
    354      * @return group
    355      */
    356     public String getGroup() {
    357         return group;
    358     }
    359
    360     /**
    361      * Mutator.  Specify a pattern from {@link java.text.SimpleDateFormat}
    362      *
    363      * @param timestampFormat
    364      */
    365     public void setTimestampFormat(String timestampFormat) {
    366         this.timestampFormat = timestampFormat;
    367     }
    368
    369     /**
    370      * Accessor
    371      *
    372      * @return timestamp format
    373      */
    374     public String getTimestampFormat() {
    375         return timestampFormat;
    376     }
    377
    378     /**
    379      * Accessor
    380      *
    381      * @return millis between retrieves of content
    382      */
    383     public long getWaitMillis() {
    384         return waitMillis;
    385     }
    386
    387     /**
    388      * Mutator
    389      *
    390      * @param waitMillis
    391      */
    392     public void setWaitMillis(long waitMillis) {
    393         this.waitMillis = waitMillis;
    394     }
    395
    396     /**
    397      * Walk the additionalLines list, looking for the EXCEPTION_PATTERN.
    398      * <p>
    399      * Return the index of the first matched line
    400      * (the match may be the 1st line of an exception)
    401      * <p>
    402      * Assumptions: <br>
    403      * - the additionalLines list may contain both message and exception lines<br>
    404      * - message lines are added to the additionalLines list and then
    405      * exception lines (all message lines occur in the list prior to all
    406      * exception lines)
    407      *
    408      * @return -1 if no exception line exists, line number otherwise
    409      */
    410     private int getExceptionLine() {
    411         for (int i = 0; i < additionalLines.size(); i++) {
    412             Matcher exceptionMatcher = exceptionPattern.matcher(additionalLines.get(i));
    413             if (exceptionMatcher.matches()) {
    414                 return i;
    415             }
    416         }
    417         return -1;
    418     }
    419
    420     /**
    421      * Combine all message lines occuring in the additionalLines list, adding
    422      * a newline character between each line
    423      * <p>
    424      * the event will already have a message - combine this message
    425      * with the message lines in the additionalLines list
    426      * (all entries prior to the exceptionLine index)
    427      *
    428      * @param firstMessageLine primary message line
    429      * @param exceptionLine    index of first exception line
    430      * @return message
    431      */
    432     private String buildMessage(String firstMessageLine, int exceptionLine) {
    433         if (additionalLines.size() == 0) {
    434             return firstMessageLine;
    435         }
    436         StringBuilder message = new StringBuilder();
    437         if (firstMessageLine != null) {
    438             message.append(firstMessageLine);
    439         }
    440
    441         int linesToProcess = (exceptionLine == -1 ? additionalLines.size() : exceptionLine);
    442
    443         for (int i = 0; i < linesToProcess; i++) {
    444             message.append(newLine);
    445             message.append(additionalLines.get(i));
    446         }
    447         return message.toString();
    448     }
    449
    450     /**
    451      * Combine all exception lines occuring in the additionalLines list into a
    452      * String array
    453      * <p>
    454      * (all entries equal to or greater than the exceptionLine index)
    455      *
    456      * @param exceptionLine index of first exception line
    457      * @return exception
    458      */
    459     private String[] buildException(int exceptionLine) {
    460         if (exceptionLine == -1) {
    461             return emptyException;
    462         }
    463         String[] exception = new String[additionalLines.size() - exceptionLine - 1];
    464         for (int i = 0; i < exception.length; i++) {
    465             exception[i] = additionalLines.get(i + exceptionLine);
    466         }
    467         return exception;
    468     }
    469
    470     /**
    471      * Construct a logging event from currentMap and additionalLines
    472      * (additionalLines contains multiple message lines and any exception lines)
    473      * <p>
    474      * CurrentMap and additionalLines are cleared in the process
    475      *
    476      * @return event
    477      */
    478     private LoggingEvent buildEvent() {
    479         if (currentMap.size() == 0) {
    480             if (additionalLines.size() > 0) {
    481                 for (Object additionalLine : additionalLines) {
    482                     getLogger().debug("found non-matching line: " + additionalLine);
    483                 }
    484             }
    485             additionalLines.clear();
    486             return null;
    487         }
    488         //the current map contains fields - build an event
    489         int exceptionLine = getExceptionLine();
    490         String[] exception = buildException(exceptionLine);
    491
    492         //messages are listed before exceptions in additionallines
    493         if (additionalLines.size() > 0 && exception.length > 0) {
    494             currentMap.put(MESSAGE, buildMessage((String) currentMap.get(MESSAGE),
    495                 exceptionLine));
    496         }
    497         LoggingEvent event = convertToEvent(currentMap, exception);
    498         currentMap.clear();
    499         additionalLines.clear();
    500         return event;
    501     }
    502
    503     /**
    504      * Read, parse and optionally tail the log file, converting entries into logging events.
    505      * <p>
    506      * A runtimeException is thrown if the logFormat pattern is malformed.
    507      *
    508      * @param bufferedReader
    509      * @throws IOException
    510      */
    511     protected void process(BufferedReader bufferedReader) throws IOException {
    512         Matcher eventMatcher;
    513         Matcher exceptionMatcher;
    514         String readLine;
    515         //if newlines are provided in the logFormat - (NL) - combine the lines prior to matching
    516         while ((readLine = bufferedReader.readLine()) != null) {
    517             StringBuilder line = new StringBuilder(readLine);
    518             //there is already one line (read above, start i at 1
    519             for (int i = 1; i < lineCount; i++) {
    520                 String thisLine = bufferedReader.readLine();
    521                 if (thisLine != null) {
    522                     line.append(newLine).append(thisLine);
    523                 }
    524             }
    525             String input = line.toString();
    526             eventMatcher = regexpPattern.matcher(input);
    527             //skip empty line entries
    528             if (input.trim().equals("")) {
    529                 continue;
    530             }
    531             exceptionMatcher = exceptionPattern.matcher(input);
    532             if (eventMatcher.matches()) {
    533                 //build an event from the previous match (held in current map)
    534                 LoggingEvent event = buildEvent();
    535                 if (event != null) {
    536                     if (passesExpression(event)) {
    537                         doPost(event);
    538                     }
    539                 }
    540                 currentMap.putAll(processEvent(eventMatcher.toMatchResult()));
    541             } else if (exceptionMatcher.matches()) {
    542                 //an exception line
    543                 additionalLines.add(input);
    544             } else {
    545                 //neither...either post an event with the line or append as additional lines
    546                 //if this was a logging event with multiple lines, each line will show up as its own event instead of being
    547                 //appended as multiple lines on the same event..
    548                 //choice is to have each non-matching line show up as its own line, or append them all to a previous event
    549                 if (appendNonMatches) {
    550                     //hold on to the previous time, so we can do our best to preserve time-based ordering if the event is a non-match
    551                     String lastTime = (String) currentMap.get(TIMESTAMP);
    552                     //build an event from the previous match (held in current map)
    553                     if (currentMap.size() > 0) {
    554                         LoggingEvent event = buildEvent();
    555                         if (event != null) {
    556                             if (passesExpression(event)) {
    557                                 doPost(event);
    558                             }
    559                         }
    560                     }
    561                     if (lastTime != null) {
    562                         currentMap.put(TIMESTAMP, lastTime);
    563                     }
    564                     currentMap.put(MESSAGE, input);
    565                 } else {
    566                     additionalLines.add(input);
    567                 }
    568             }
    569         }
    570
    571         //process last event if one exists
    572         LoggingEvent event = buildEvent();
    573         if (event != null) {
    574             if (passesExpression(event)) {
    575                 doPost(event);
    576             }
    577         }
    578     }
    579
    580     protected void createPattern() {
    581         regexpPattern = Pattern.compile(regexp);
    582     }
    583
    584     /**
    585      * Helper method that supports the evaluation of the expression
    586      *
    587      * @param event
    588      * @return true if expression isn't set, or the result of the evaluation otherwise
    589      */
    590     private boolean passesExpression(LoggingEvent event) {
    591         if (event != null) {
    592             if (expressionRule != null) {
    593                 return (expressionRule.evaluate(event, null));
    594             }
    595         }
    596         return true;
    597     }
    598
    599     /**
    600      * Convert the match into a map.
    601      * <p>
    602      * Relies on the fact that the matchingKeywords list is in the same
    603      * order as the groups in the regular expression
    604      *
    605      * @param result
    606      * @return map
    607      */
    608     private Map processEvent(MatchResult result) {
    609         Map map = new HashMap();
    610         //group zero is the entire match - process all other groups
    611         for (int i = 1; i < result.groupCount() + 1; i++) {
    612             Object key = matchingKeywords.get(i - 1);
    613             Object value = result.group(i);
    614             map.put(key, value);
    615
    616         }
    617         return map;
    618     }
    619
    620     /**
    621      * Helper method that will convert timestamp format to a pattern
    622      *
    623      * @return string
    624      */
    625     private String convertTimestamp() {
    626         //some locales (for example, French) generate timestamp text with characters not included in \w -
    627         // now using \S (all non-whitespace characters) instead of /w
    628         String result = "";
    629         if (timestampFormat != null) {
    630             result = timestampFormat.replaceAll(Pattern.quote("+"), "[+]");
    631             result = result.replaceAll(VALID_DATEFORMAT_CHAR_PATTERN, "\\\\S+");
    632             //make sure dots in timestamp are escaped
    633             result = result.replaceAll(Pattern.quote("."), "\\\\.");
    634         }
    635         return result;
    636     }
    637
    638     protected void setHost(String host) {
    639         this.host = host;
    640     }
    641
    642     protected void setPath(String path) {
    643         this.path = path;
    644     }
    645
    646     public String getPath() {
    647         return path;
    648     }
    649
    650     /**
    651      * Build the regular expression needed to parse log entries
    652      */
    653     protected void initialize() {
    654         if (host == null && path == null) {
    655             try {
    656                 URL url = new URL(fileURL);
    657                 host = url.getHost();
    658                 path = url.getPath();
    659             } catch (MalformedURLException e1) {
    660                 // TODO Auto-generated catch block
    661                 e1.printStackTrace();
    662             }
    663         }
    664         if (host == null || host.trim().equals("")) {
    665             host = DEFAULT_HOST;
    666         }
    667         if (path == null || path.trim().equals("")) {
    668             path = fileURL;
    669         }
    670
    671         currentMap = new HashMap();
    672         additionalLines = new ArrayList<>();
    673         matchingKeywords = new ArrayList<>();
    674
    675         if (timestampFormat != null) {
    676             dateFormat = new SimpleDateFormat(quoteTimeStampChars(timestampFormat));
    677             timestampPatternText = convertTimestamp();
    678         }
    679         //if custom level definitions exist, parse them
    680         updateCustomLevelDefinitionMap();
    681         try {
    682             if (filterExpression != null) {
    683                 expressionRule = ExpressionRule.getRule(filterExpression);
    684             }
    685         } catch (Exception e) {
    686             getLogger().warn("Invalid filter expression: " + filterExpression, e);
    687         }
    688
    689         List<String> buildingKeywords = new ArrayList<>();
    690
    691         String newPattern = logFormat;
    692
    693         //process newlines - (NL) - in the logFormat - before processing properties
    694         int index = 0;
    695         while (index > -1) {
    696             index = newPattern.indexOf(NEWLINE);
    697             if (index > -1) {
    698                 //keep track of number of expected newlines in the format, so the lines can be concatenated prior to matching
    699                 lineCount++;
    700                 newPattern = singleReplace(newPattern, NEWLINE, NEWLINE_REGEXP);
    701             }
    702         }
    703
    704         String current = newPattern;
    705         //build a list of property names and temporarily replace the property with an empty string,
    706         //we'll rebuild the pattern later
    707         List<String> propertyNames = new ArrayList<>();
    708         index = 0;
    709         while (index > -1) {
    710             if (current.contains(PROP_START) && current.contains(PROP_END)) {
    711                 index = current.indexOf(PROP_START);
    712                 String longPropertyName = current.substring(current.indexOf(PROP_START), current.indexOf(PROP_END) + 1);
    713                 String shortProp = getShortPropertyName(longPropertyName);
    714                 buildingKeywords.add(shortProp);
    715                 propertyNames.add(longPropertyName);
    716                 current = current.substring(longPropertyName.length() + 1 + index);
    717                 newPattern = singleReplace(newPattern, longPropertyName, Integer.toString(buildingKeywords.size() - 1));
    718             } else {
    719                 //no properties
    720                 index = -1;
    721             }
    722         }
    723
    724         /*
    725          * we're using a treemap, so the index will be used as the key to ensure
    726          * keywords are ordered correctly
    727          *
    728          * examine pattern, adding keywords to an index-based map patterns can
    729          * contain only one of these per entry...properties are the only 'keyword'
    730          * that can occur multiple times in an entry
    731          */
    732         for (Object keyword1 : keywords) {
    733             String keyword = (String) keyword1;
    734             int index2 = newPattern.indexOf(keyword);
    735             if (index2 > -1) {
    736                 buildingKeywords.add(keyword);
    737                 newPattern = singleReplace(newPattern, keyword, Integer.toString(buildingKeywords.size() - 1));
    738             }
    739         }
    740
    741         StringBuilder buildingInt = new StringBuilder();
    742
    743         for (int i = 0; i < newPattern.length(); i++) {
    744             String thisValue = String.valueOf(newPattern.substring(i, i + 1));
    745             if (isInteger(thisValue)) {
    746                 buildingInt.append(thisValue);
    747             } else {
    748                 String stringInt = buildingInt.toString();
    749                 if (isInteger(stringInt)) {
    750                     matchingKeywords.add(buildingKeywords.get(Integer.parseInt(stringInt)));
    751                 }
    752                 //reset
    753                 buildingInt.setLength(0);
    754             }
    755         }
    756
    757         //if the very last value is an int, make sure to add it
    758         String stringInt = buildingInt.toString();
    759         if (isInteger(stringInt)) {
    760             matchingKeywords.add(buildingKeywords.get(Integer.parseInt(stringInt)));
    761         }
    762
    763         newPattern = replaceMetaChars(newPattern);
    764
    765         //compress one or more spaces in the pattern into the [ ]+ regexp
    766         //(supports padding of level in log files)
    767         newPattern = newPattern.replaceAll(MULTIPLE_SPACES_REGEXP, MULTIPLE_SPACES_REGEXP);
    768         newPattern = newPattern.replaceAll(Pattern.quote(PATTERN_WILDCARD), REGEXP_DEFAULT_WILDCARD);
    769         //use buildingKeywords here to ensure correct order
    770         for (int i = 0; i < buildingKeywords.size(); i++) {
    771             String keyword = buildingKeywords.get(i);
    772             //make the final keyword greedy (we're assuming it's the message)
    773             if (i == (buildingKeywords.size() - 1)) {
    774                 newPattern = singleReplace(newPattern, String.valueOf(i), GREEDY_GROUP);
    775             } else if (TIMESTAMP.equals(keyword)) {
    776                 newPattern = singleReplace(newPattern, String.valueOf(i), "(" + timestampPatternText + ")");
    777             } else if (LOGGER.equals(keyword) || LEVEL.equals(keyword)) {
    778                 newPattern = singleReplace(newPattern, String.valueOf(i), NOSPACE_GROUP);
    779             } else {
    780                 newPattern = singleReplace(newPattern, String.valueOf(i), DEFAULT_GROUP);
    781             }
    782         }
    783
    784         regexp = newPattern;
    785         getLogger().debug("regexp is " + regexp);
    786     }
    787
    788     private void updateCustomLevelDefinitionMap() {
    789         if (customLevelDefinitions != null) {
    790             StringTokenizer entryTokenizer = new StringTokenizer(customLevelDefinitions, ",");
    791
    792             customLevelDefinitionMap.clear();
    793             while (entryTokenizer.hasMoreTokens()) {
    794                 StringTokenizer innerTokenizer = new StringTokenizer(entryTokenizer.nextToken(), "=");
    795                 customLevelDefinitionMap.put(innerTokenizer.nextToken(), Level.toLevel(innerTokenizer.nextToken()));
    796             }
    797         }
    798     }
    799
    800     private boolean isInteger(String value) {
    801         try {
    802             Integer.parseInt(value);
    803             return true;
    804         } catch (NumberFormatException nfe) {
    805             return false;
    806         }
    807     }
    808
    809     private String quoteTimeStampChars(String input) {
    810         //put single quotes around text that isn't a supported dateformat char
    811         StringBuilder result = new StringBuilder();
    812         //ok to default to false because we also check for index zero below
    813         boolean lastCharIsDateFormat = false;
    814         for (int i = 0; i < input.length(); i++) {
    815             String thisVal = input.substring(i, i + 1);
    816             boolean thisCharIsDateFormat = VALID_DATEFORMAT_CHARS.contains(thisVal);
    817             //we have encountered a non-dateformat char
    818             if (!thisCharIsDateFormat && (i == 0 || lastCharIsDateFormat)) {
    819                 result.append("'");
    820             }
    821             //we have encountered a dateformat char after previously encountering a non-dateformat char
    822             if (thisCharIsDateFormat && i > 0 && !lastCharIsDateFormat) {
    823                 result.append("'");
    824             }
    825             lastCharIsDateFormat = thisCharIsDateFormat;
    826             result.append(thisVal);
    827         }
    828         //append an end single-quote if we ended with non-dateformat char
    829         if (!lastCharIsDateFormat) {
    830             result.append("'");
    831         }
    832         return result.toString();
    833     }
    834
    835     private String singleReplace(String inputString, String oldString, String newString) {
    836         int propLength = oldString.length();
    837         int startPos = inputString.indexOf(oldString);
    838         if (startPos == -1) {
    839             getLogger().info("string: " + oldString + " not found in input: " + inputString + " - returning input");
    840             return inputString;
    841         }
    842         if (startPos == 0) {
    843             inputString = inputString.substring(propLength);
    844             inputString = newString + inputString;
    845         } else {
    846             inputString = inputString.substring(0, startPos) + newString + inputString.substring(startPos + propLength);
    847         }
    848         return inputString;
    849     }
    850
    851     private String getShortPropertyName(String longPropertyName) {
    852         String currentProp = longPropertyName.substring(longPropertyName.indexOf(PROP_START));
    853         String prop = currentProp.substring(0, currentProp.indexOf(PROP_END) + 1);
    854         String shortProp = prop.substring(PROP_START.length(), prop.length() - 1);
    855         return shortProp;
    856     }
    857
    858     /**
    859      * Some perl5 characters may occur in the log file format.
    860      * Escape these characters to prevent parsing errors.
    861      *
    862      * @param input
    863      * @return string
    864      */
    865     private String replaceMetaChars(String input) {
    866         //escape backslash first since that character is used to escape the remaining meta chars
    867         input = input.replaceAll("\\\\", "\\\\\\");
    868
    869         //don't escape star - it's used as the wildcard
    870         input = input.replaceAll(Pattern.quote("]"), "\\\\]");
    871         input = input.replaceAll(Pattern.quote("["), "\\\\[");
    872         input = input.replaceAll(Pattern.quote("^"), "\\\\^");
    873         input = input.replaceAll(Pattern.quote("$"), "\\\\$");
    874         input = input.replaceAll(Pattern.quote("."), "\\\\.");
    875         input = input.replaceAll(Pattern.quote("|"), "\\\\|");
    876         input = input.replaceAll(Pattern.quote("?"), "\\\\?");
    877         input = input.replaceAll(Pattern.quote("+"), "\\\\+");
    878         input = input.replaceAll(Pattern.quote("("), "\\\\(");
    879         input = input.replaceAll(Pattern.quote(")"), "\\\\)");
    880         input = input.replaceAll(Pattern.quote("-"), "\\\\-");
    881         input = input.replaceAll(Pattern.quote("{"), "\\\\{");
    882         input = input.replaceAll(Pattern.quote("}"), "\\\\}");
    883         input = input.replaceAll(Pattern.quote("#"), "\\\\#");
    884         return input;
    885     }
    886
    887     /**
    888      * Convert a keyword-to-values map to a LoggingEvent
    889      *
    890      * @param fieldMap
    891      * @param exception
    892      * @return logging event
    893      */
    894     private LoggingEvent convertToEvent(Map fieldMap, String[] exception) {
    895         if (fieldMap == null) {
    896             return null;
    897         }
    898
    899         //a logger must exist at a minimum for the event to be processed
    900         if (!fieldMap.containsKey(LOGGER)) {
    901             fieldMap.put(LOGGER, "Unknown");
    902         }
    903         if (exception == null) {
    904             exception = emptyException;
    905         }
    906
    907         Logger logger;
    908         long timeStamp = 0L;
    909         String level;
    910         String threadName;
    911         Object message;
    912         String ndc;
    913         String className;
    914         String methodName;
    915         String eventFileName;
    916         String lineNumber;
    917         Hashtable properties = new Hashtable();
    918
    919         logger = Logger.getLogger((String) fieldMap.remove(LOGGER));
    920
    921         if ((dateFormat != null) && fieldMap.containsKey(TIMESTAMP)) {
    922             try {
    923                 timeStamp = dateFormat.parse((String) fieldMap.remove(TIMESTAMP))
    924                     .getTime();
    925             } catch (Exception e) {
    926                 e.printStackTrace();
    927             }
    928         }
    929         //use current time if timestamp not parseable/dateformat not specified
    930         if (timeStamp == 0L) {
    931             timeStamp = System.currentTimeMillis();
    932         }
    933
    934         message = fieldMap.remove(MESSAGE);
    935         if (message == null) {
    936             message = "";
    937         }
    938
    939         level = (String) fieldMap.remove(LEVEL);
    940         Level levelImpl;
    941         if (level == null) {
    942             levelImpl = Level.DEBUG;
    943         } else {
    944             //first try to resolve against custom level definition map, then fall back to regular levels
    945             levelImpl = customLevelDefinitionMap.get(level);
    946             if (levelImpl == null) {
    947                 levelImpl = Level.toLevel(level.trim());
    948                 if (!level.equals(levelImpl.toString())) {
    949                     //check custom level map
    950                     if (levelImpl == null) {
    951                         levelImpl = Level.DEBUG;
    952                         getLogger().debug("found unexpected level: " + level + ", logger: " + logger.getName() + ", msg: " + message);
    953                         //make sure the text that couldn't match a level is added to the message
    954                         message = level + " " + message;
    955                     }
    956                 }
    957             }
    958         }
    959
    960         threadName = (String) fieldMap.remove(THREAD);
    961
    962         ndc = (String) fieldMap.remove(NDC);
    963
    964         className = (String) fieldMap.remove(CLASS);
    965
    966         methodName = (String) fieldMap.remove(METHOD);
    967
    968         eventFileName = (String) fieldMap.remove(FILE);
    969
    970         lineNumber = (String) fieldMap.remove(LINE);
    971
    972         properties.put(Constants.HOSTNAME_KEY, host);
    973         properties.put(Constants.APPLICATION_KEY, path);
    974         properties.put(Constants.RECEIVER_NAME_KEY, getName());
    975         if (group != null) {
    976             properties.put(Constants.GROUP_KEY, group);
    977         }
    978
    979         //all remaining entries in fieldmap are properties
    980         properties.putAll(fieldMap);
    981
    982         LocationInfo info;
    983
    984         if ((eventFileName != null) || (className != null) || (methodName != null)
    985             || (lineNumber != null)) {
    986             info = new LocationInfo(eventFileName, className, methodName, lineNumber);
    987         } else {
    988             info = LocationInfo.NA_LOCATION_INFO;
    989         }
    990
    991         LoggingEvent event = new LoggingEvent(null,
    992             logger, timeStamp, levelImpl, message,
    993             threadName,
    994             new ThrowableInformation(exception),
    995             ndc,
    996             info,
    997             properties);
    998
    999         return event;
    1000     }
    1001
    1002 //  public static void main(String[] args) {
    1003 //    org.apache.log4j.Logger rootLogger = org.apache.log4j.Logger.getRootLogger();
    1004 //    org.apache.log4j.ConsoleAppender appender = new org.apache.log4j.ConsoleAppender(new org.apache.log4j.SimpleLayout());
    1005 //    appender.setName("console");
    1006 //    rootLogger.addAppender(appender);
    1007 //    LogFilePatternReceiver test = new LogFilePatternReceiver();
    1008 //    org.apache.log4j.spi.LoggerRepository repo = new org.apache.log4j.LoggerRepositoryExImpl(org.apache.log4j.LogManager.getLoggerRepository());
    1009 //    test.setLoggerRepository(repo);
    1010 //    test.setLogFormat("PROP(RELATIVETIME) [THREAD] LEVEL LOGGER * - MESSAGE");
    1011 //    test.setTailing(false);
    1012 //    test.setAppendNonMatches(true);
    1013 //    test.setTimestampFormat("yyyy-MM-d HH:mm:ss,SSS");
    1014 //    test.setFileURL("file:///C:/log/test.log");
    1015 //    test.activateOptions();
    1016 //  }
    1017
    1018     /**
    1019      * Close the reader.
    1020      */
    1021     public void shutdown() {
    1022         getLogger().info(getPath() + " shutdown");
    1023         active = false;
    1024         try {
    1025             if (reader != null) {
    1026                 reader.close();
    1027                 reader = null;
    1028             }
    1029         } catch (IOException ioe) {
    1030             ioe.printStackTrace();
    1031         }
    1032     }
    1033
    1034     /**
    1035      * Read and process the log file.
    1036      */
    1037     public void activateOptions() {
    1038         getLogger().info("activateOptions");
    1039         active = true;
    1040         Runnable runnable = new Runnable() {
    1041             public void run() {
    1042                 initialize();
    1043                 while (reader == null) {
    1044                     getLogger().info("attempting to load file: " + getFileURL());
    1045                     try {
    1046                         reader = new InputStreamReader(new URL(getFileURL()).openStream(), "UTF-8");
    1047                     } catch (FileNotFoundException fnfe) {
    1048                         getLogger().info("file not available - will try again");
    1049                         synchronized (this) {
    1050                             try {
    1051                                 wait(MISSING_FILE_RETRY_MILLIS);
    1052                             } catch (InterruptedException ie) {
    1053                             }
    1054                         }
    1055                     } catch (IOException ioe) {
    1056                         getLogger().warn("unable to load file", ioe);
    1057                         return;
    1058                     }
    1059                 }
    1060                 try {
    1061                     BufferedReader bufferedReader = new BufferedReader(reader);
    1062                     createPattern();
    1063                     do {
    1064                         process(bufferedReader);
    1065                         try {
    1066                             synchronized (this) {
    1067                                 wait(waitMillis);
    1068                             }
    1069                         } catch (InterruptedException ie) {
    1070                         }
    1071                         if (tailing) {
    1072                             getLogger().debug("tailing file");
    1073                         }
    1074                     } while (tailing);
    1075
    1076                 } catch (IOException ioe) {
    1077                     //io exception - probably shut down
    1078                     getLogger().info("stream closed");
    1079                 }
    1080                 getLogger().debug("processing " + path + " complete");
    1081                 shutdown();
    1082             }
    1083         };
    1084         if (useCurrentThread) {
    1085             runnable.run();
    1086         } else {
    1087             new Thread(runnable, "LogFilePatternReceiver-" + getName()).start();
    1088         }
    1089     }
    1090 }