lwatch.c

Go to the documentation of this file.

/*****************************************************************************/
/******************************** Documentation ******************************/
/*****************************************************************************/

/**
 * \mainpage
 *
 * See \ref LogWatchOverview for an overview of how log works and how the
 * the architectural components fit together.
 *
 * <br><br>
 * See \ref InotifyOverview for an overview of how Inotify is utilized to
 * provide a monitoring backend for the log watcher.
 *
 * Click on the files tab to visit the documentation for each file. Click on the
 * code link to see the implementation or click on "Go to the source code of
 * this file." link when viewing file documentation. Clicking on almost any item
 * takes you either to the documentation for that item or it's implementation.
 */

/**
 * \file
 * Implements the basic watching logic for log files.
 *
 * \page LogWatchOverview Log Watch Overview
 *
 * \section WatchArchitectureDoc How the architectural pieces fit together.
 *
 * The primary purpose of the log watch daemon is to collect data from a set of
 * log files and transmit it to a central server. The act of harvesting the data
 * from a log file is called a "reap". To accomplish this we must have the
 * following set of logical components:
 * - Configuration
 *   - Which files to reap and how
 * - Watcher
 *   - Watches a file for state changes
 *   - Logically divided into
 *     - Common watch code (lwatch)
 *     - One or more implementation specific watch backends. Some OS's have
 *       facilities for monitoring file changes and we utilize the OS specifc
 *       monitoring whenever possible because it's the most efficient. If no OS
 *       support is available for file monitoring we can fallback on polling. At
 *       the moment the following backends are implemented:
 *       - Inotify backend
 * - Persistent state storage
 *   - The log watch daemon will start and stop (and hopefully never
 *     crash). However there is nothing preventing the file system from being
 *     modified during any interval of time the log watch daemon is not running.
 *     Therefore we must store the state of every file we're watching in
 *     persistent storage so that we know things such as what data for the file
 *     has already been reaped and sent to the central server, what constitutes
 *     unreaped data, has the file been overwritten, etc. We store this
 *     information in a SQLite database. When the log watch daemon starts up it
 *     consults the database and examines the file system to determine what
 *     actions it must take. During run time the database is used to "cache"
 *     information about a file or a watch.
 * - Central server communications
 *   - When data for a file is reaped it must be sent to the central server.
 *   - The log watch daemon is also responsible for sending log events (ELAPI)
 *     to the central server sent to it from applications running locally.
 *
 * \section HowFilesAreWatchedDoc How files are watched.
 *
 * When a file is asked to be watched by the user it is called a watch target. A
 * watch target does not need to exist, the log watcher must be capable of
 * watching for when it comes into existence and when it no longer exists
 * without error and to be able to reap all it's data whenever it is in
 * existence.
 *
 * The establishment of a watch target effectively constitutes a watch on a
 * logical log file which consists of a primary log file and it's associated set
 * of backups (see \ref VocabularyDoc). The backup log files must also be
 * watched in addition to the primary file because all the files in a logical
 * log file has data which needs to be reaped.
 *
 * There is a further complication introduced by the presence of log rotation,
 * backups, and the behavior of the UNIX file system. In UNIX a path name is a
 * weak association to the contents of a file. File contents in UNIX are
 * actaully identified by an inode. In fact it is entirely possible for more
 * than one path name to point to the same inode. Another consequence is a file
 * can be opened under one path name (OS locates the inode and returns a file
 * descriptor handle to the file contents) and while the process has that file
 * open it's path name can be modified such that it no longer has the same path
 * name as when it was opened but the application is still reading or writing
 * the same file contents for the same inode but now under a new pathname.
 *
 * Typically when a log file is rotated the rotation is performed by a process
 * other than the application (e.g. the logrotate daemon). Logrotate will rename
 * the primary file (the one the application has open) to a log backup
 * name. Logrotate may optionally signal the application after performing the
 * rotation that it should close it's log file and reopened it again under the
 * primary name. Or it may wait for the application to close and reopen the
 * primary file at some future point in time (some applications open and close
 * their log files after every write). The critical issue for us is that we
 * cannot just watch the primary file, after a rotation the application may
 * continue to write into what is now a backup path name because it is unaware
 * the path name of the file it has a file descriptor open on has been
 * subsequently renamed, it thinks it's still writing to the primary file. Thus
 * we must watch all logical log files (primary plus backups). We also need to
 * watch backups because during an interval of time when the log watch daemon is
 * not running a rotation may have occured which we did not receive an event
 * for.
 *
 * It is usually not sufficient to ask a file monitoring backend to monitor a
 * file because operations such as file creation, file deletion, and file
 * renames occur on the directory containing the file, not on the file
 * itself. As a consequence of this we ONLY MONITOR DIRECTORIES. Each directory
 * is assigned a set of files to watch for.
 *
 * If the directory containing a watched file does not yet exist we can't set a
 * watch on it because watches can only be established on file system objects
 * which exist. Therefore the watch on a directory may also be given the
 * responsibility to watch not only for files it may contain but also for the
 * creation of any sub-directories which might lead to a watched file
 * (descendant watch).
 *
 * Thus in order to watch changes in a requested target we may also end up
 * watching:
 * - any directory containing a target
 * - any directory containing a backup of the target
 * - any directory which is the ancestor of a directory needing to be watched
 *   but which does not yet exist.
 *
 * Just to make life fun the set of watches established to accmodate the above
 * need to be continuously adjusted because the file system is constantly
 * changing underneath us. If a directory containing a watched file is deleted
 * we need to move the watch to one of it's ancestors. If a directory is created
 * which should have been watched but wasn't because it didn't exist then we
 * need to move the watch from it's ancestor to the closest existing
 * ancestor. The act of deciding a watched file has new data appended to it
 * which needs to be reaped is actually one of the simpler tasks in log
 * watching.
 *
 * Please see \ref DescendantWatchesDoc for further explanation of how watch
 * points, descendants, and non-existent entities are handled.
 *
 * \section IndependentWatchBackEndDoc Independent backends for watching.
 *
 * Different OS's have different support for monitoring file system
 * modifications. We need to be portable across many different OS's but we also
 * need to be as efficient and robust as possible. Therefore we abstract out the
 * OS specific mechanism for file system monitoring and put it into a monitoring
 * "back end". The monitoring backend sends generic platform independent
 * monitoring events to the log watcher. The log watcher dispaches on these
 * generic monitoring events. Thus we can deploy the log watch daemon in a
 * variety environments and configure it use the optimal monitoring backend for
 * the system. If an OS does not provide file system monitoring facilities an
 * backend can be written which relies on polling the file system. A polling
 * backend is less than ideal because it may miss file system events which occur
 * between polling and because it's inherently inefficient, it would have to
 * check every watched item irrespective of whether it actually changed and
 * repeat this every poll interval. Clearly it's more efficient and robust to use
 * an OS based file monitoring facility which sends events only when things
 * change.
 *
 * The basic job of a file monitoring backend is to accept requests to watch
 * specific objects and when a change occurs on that object to formulate it as a
 * generic platform neutral event and dispatch that event to the log watcher.
 *
 * \section WatchEventDoc How watch events are utilized.
 *
 * \see lwatch_event_t for a description of the generic log watch event structure.
 *
 * The set of possible generic log watch events types are:
 * <DL>
 * <DT> LWATCH_EVENT_CREATE </DT>
 * <DD>
 * a file or directory was created
 * </DD>
 * <DT> LWATCH_EVENT_DELETE </DT>
 * <DD>
 * a file or directory was deleted
 * </DD>
 * <DT> LWATCH_EVENT_RENAME </DT>
 * <DD>
 * a file or directory was renamed
 * </DD>
 * <DT> LWATCH_EVENT_MODIFY </DT>
 * <DD>
 * a file or directory was modified
 * </DD>
 * <DT> LWATCH_EVENT_OPEN </DT>
 * <DD>
 * a file or directory was opened
 * </DD>
 * <DT> LWATCH_EVENT_CLOSE </DT>
 * <DD>
 * a file or directory was closed
 * </DD>
 * </DL>
 *
 * \section VocabularyDoc Vocabulary Terms
 * <DL>
 * <DT> path watch object </DT>
 * <DD>
 * In the inotify backend watches are established on path names. Inotify assigns
 * the watch an id. When Inotify sends an event it identifies which watch the
 * event is associated with via the id. The path watch object encapsultes both
 * the id and the path name into one object.
 * </DD>
 * <DT> watch target </DT>
 * <DD>
 * A file we've been explicity asked to watch by the user. In the course of
 * running we may watch other directories and files in addition to the explicit
 * targets requested in order to fulfill other needs. To distinguish between
 * watches requested by the user and watches needed for other purposes we refer
 * to user requested watches as a "watch target" because it's a explicit target
 * established by the user.
 * </DD>
 * <DT> descendant watch </DT>
 * <DD>
 * Inotify can watch both directories and files. Watch targets are files only
 * thus one might expect we would establish an Inotify watch on a file. However
 * it is the directory containing the file which reports the file's creation,
 * deletion, and renaming. In addition an Inotify watch on a directory will
 * report modifications to a file just as if the watch had been established on
 * the file directly. Also critically import is the fact Inotify watches can
 * only be established on file system objects which exist. These constraints
 * mean we only establish Inotify watches on directories. An Inotify watch on a
 * directory is responsible for watching for every file it contains which is a
 * watched file AND for any files below it whose directory structure does not
 * yet exist. These are descendants of the directory being watched. Technically
 * files contained in the directory are also descendants of the directory if one
 * considers a path name as a sequence of path components. A descendant watch
 * then is any file contained in the directory being watched or any file below
 * the directory which should be watched for it's creation. This means an
 * Inotify path watch object (i.e. watch point) has a set of descendant watches
 * it is responsible for.
 * </DD>
 * <DT> watch point </DT>
 * <DD>
 * Any directory which has one or more descendant watches (either because it
 * holds a file being watched or because it's along the path to non-existent
 * directory) is called a watch point. This is because the watch backend is
 * instructed to watch that directory for any of it's descendant watches.
 * </DD>
 * <DT> closest existing ancestor </DT>
 * <DD>
 * When asked to watch a file we actually establish a watch on the directory
 * containing the file. But what if that directory doesn't exist? It means we
 * have to start at the last directory in the chain of directories comprising
 * the path and start popping off directory components progressively toward the
 * root of the file system until we find a directory which exists in the file
 * system, we call this the "closest existing ancestor".
 * </DD>
 * <DT> reap </DT>
 * <DD>
 * The primary task of the log watch daemon is to collect data written to log
 * files and transmit them to a central server for storage and analysis. The
 * term "reap" means to collect or harvest. Thus when we say "reap" we mean the
 * act of collecting or harvesting the log data for transmission to the central
 * server.
 * </DD>
 * <DT> log rotation </DT>
 * <DD>
 * Log files are typically rotated meaning that when they reach a threshold on
 * size or time the log file is moved to a backup name. Typically the N youngest
 * backups are kept, each time a file is rotated the oldest backup is deleted to
 * make room the new backup so there are most N backups at any given time. Think
 * of it as a FIFO.
 * </DD>
 * <DT> primary file </DT>
 * <DD>
 * A primary file is the log file before being rotated. The primary file and the
 * set of rotated backups derived from the primary file constitute the same
 * logical log file, but partitioned into different physical files.
 * </DD>
 * <DT> backup file </DT>
 * <DD>
 * A backup file was a primary file prior to rotation. A backup file always has
 * exactly one primary file that it was derived from. There may be many backups
 * who share a single common primary file. One primary file and a collection of
 * zero or more backups derived from the primary constitute a logical log file.
 * </DD>
 * <DT> logical log file </DT>
 * <DD>
 * A single primary and a collection of zero or more backups derived from the
 * primary constitute a logical log file. This is because the primary and the
 * ordered sequence of backups represent a single continuous data stream which
 * happen to be partitioned into distinct physical files in the file system.
 * </DD>
 * </DL>
 */

/*****************************************************************************/
/******************************* Include Files *******************************/
/*****************************************************************************/

#define _GNU_SOURCE             /**< Turn on GNU extensions */
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <getopt.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/inotify.h>
#include <sys/select.h>
#include <signal.h>

#include "config.h"
#include "dhash.h"
#include "path_utils.h"
#include "logging.h"
#include "regexp.h"
#include "inotify_watch.h"
#include "util.h"
#include "lwatch.h"

/*****************************************************************************/
/****************************** Internal Defines *****************************/
/*****************************************************************************/

#define LOG_LEVEL_ARG            256 /**< log_level            command line argument */
#define LOG_TO_CONSOLE_ARG       257 /**< log_to_console       command line argument */
#define LOGFILE_PATH_ARG         258 /**< logfile_path         command line argument */
#define NO_LOGGING_TO_FILE_ARG   259 /**< no_logging_to_file   command line argument */

#define LOG_SHOW_LEVEL_ARG       260 /**< log_show_level       command line argument */
#define LOG_NO_SHOW_LEVEL_ARG    261 /**< log_no_show_level    command line argument */
#define LOG_SHOW_FILE_ARG        262 /**< log_show_file        command line argument */
#define LOG_NO_SHOW_FILE_ARG     263 /**< log_no_show_file     command line argument */
#define LOG_SHOW_LINE_ARG        264 /**< log_show_line        command line argument */
#define LOG_NO_SHOW_LINE_ARG     265 /**< log_no_show_line     command line argument */
#define LOG_SHOW_FUNCTION_ARG    266 /**< log_show_function    command line argument */
#define LOG_NO_SHOW_FUNCTION_ARG 267 /**< log_no_show_function command line argument */
#define LOG_SHOW_TIME_ARG        268 /**< log_show_time        command line argument */
#define LOG_NO_SHOW_TIME_ARG     269 /**< log_no_show_time     command line argument */

/*****************************************************************************/
/************************** Internal Type Definitions ************************/
/*****************************************************************************/

/**
 * Operational program modes, e.g. what the user wants the program to do.
 */
typedef enum {
    PROG_ACTION_MONITOR,        /**< run in daemon mode monitoring files */
    PROG_ACTION_LIST,           /**< dump the contents of file watch database */
    PROG_ACTION_FIND,           /**< locate log files */
} prog_action_t;

/**
 * Possible states of a file we're monitoring.
 */
typedef enum {
    FILE_NO_CHANGE,             /**< file has not changed */
    FILE_APPENDED,              /**< file had data appended to it */
    FILE_REPLACED,              /**< file had it's contents replaced */
} file_event_t;

/*****************************************************************************/
/**********************  External Function Declarations  *********************/
/*****************************************************************************/

/*****************************************************************************/
/**********************  Internal Function Declarations  *********************/
/*****************************************************************************/

static const char *file_event_string(file_event_t event_type);
static int reap_file_data(struct lwatch_path_info_t *path_info);
static int get_create_path_info(const char *path, struct lwatch_path_info_t *path_info);
static int process_file_modification(const char *path);
static int process_file_close(const char *path);
static int process_file_create(const char *path);
static int process_file_delete(const char *path);
static int process_file_rename(const char *prev_path, const char *new_path);
static int regexp_init(void);
static int regexp_fini(void);
static int lwatch_init(void);
static int lwatch_fini(void);
static void signal_handler (int signum);
static const char *lwatch_event_type_string(lwatch_event_type event_type);
static const char *lwatch_event_string(struct lwatch_event_t *event);
static int lwatch_event_callback(struct lwatch_event_t *event);
static int start_monitoring(const char *path_arg);
static int compare_stat_info(struct stat *stat_info, struct lwatch_path_info_t *path_info, file_event_t *result_arg);
static int get_path_info_reap_candidacy(struct lwatch_path_info_t *path_info, bool *should_reap);
static int set_path_info_reap_candidacy(struct lwatch_path_info_t *path_info);
static int update_stat_info_and_write(struct stat *cur_stat_info, struct lwatch_path_info_t *path_info);
static int dump_database(void);
static int validate_backups_in_database(void);
static int process_pending_reaps(void);
static int main_loop(void);

/*****************************************************************************/
/*************************  External Global Variables  ***********************/
/*****************************************************************************/

/*****************************************************************************/
/*************************  Internal Global Variables  ***********************/
/*****************************************************************************/

/**
 * regular expression which matches a numbered backup path name
 * match 0 = full match
 * match 1 = directory
 * match 2 = basename
 * match 3 = number
 * match 4 = extension
 */
struct regexp_t logrotate_numbered_backup_regexp = {"(.*/)?([^/]+)\\.(\\d+)(\\.gz|.\\bz2)?$"};

/**
 * regular expression which matches a dated backup path name
 * match 0 = full match
 * match 1 = directory
 * match 2 = basename
 * match 3 = full date
 * match 4 = year
 * match 5 = month
 * match 6 = day
 * match 7 = extension
 */
struct regexp_t logrotate_date_backup_regexp = {"(.*/)?([^/]+)-((\\d{4})(\\d{2})(\\d{2}))(\\.gz|.\\bz2)?$"};

/**
 * How often to reap log file updates (number of seconds)
 */
static int reap_interval = 15;

/*****************************************************************************/
/****************************  Inline Functions  *****************************/
/*****************************************************************************/

/*****************************************************************************/
/***************************  Internal Functions  ****************************/
/*****************************************************************************/

/**
 * Translate a file_event_t enumeration into a string.
 *
 * \param[in] event_type The event type enumeartion whose string representation is sought.
 * \return           Pointer to thread local static string
 */
static const char *file_event_string(file_event_t event_type)
{
    static __thread char buf[80]; /* thread local static buffer */

    switch(event_type) {
    case FILE_NO_CHANGE: return _("No change");
    case FILE_APPENDED:  return _("Appended");
    case FILE_REPLACED:  return _("Replaced");
    default:
        snprintf(buf, sizeof(buf), "unknown(%d)", event_type);
        return buf;
    }

}

/**
 * Given a path name lookup it's path_info, if it doesn't exist create it
 *
 * \param[in]  path      The path name to lookup.
 * \param[out] path_info Pointer to path_info which will be initialized with result.
 * \return               SUCCESS (0) or non-zero error code otherwise
 */
static int get_create_path_info(const char *path, struct lwatch_path_info_t *path_info)
{
    int error;

    error = database_lookup_path_info(path, path_info);
    if ((error != SUCCESS) && (error != WATCH_DATABASE_ERROR_NOT_FOUND)) {
        log_msg(LOG_ERROR, _("database_lookup_path_info failed \"%s\" (%s)\n"),
                path, error_string(error));
        return error;
    }

    /* Did not exist in database, create a new path_info, intialize it, write it into the database */
    if (error == WATCH_DATABASE_ERROR_NOT_FOUND) {
        if ((error = init_path_info(path_info, path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("could not initialize path_info for path \"%s\" (%s)\n"),
                    path, error_string(error));
            return error;
        }

        if ((error = database_write_path_info(path_info)) != SUCCESS) {
            log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                    path_info->path, error_string(error));
            return error;
        }
    }

    return SUCCESS;
}

/**
 * A file has been modified, record that it will need to be reaped.
 *
 * \param[in] path Path name of the watch target to take action on.
 * \return     SUCCESS (0) or non-zero error code otherwise
 */
static int process_file_modification(const char *path)
{
    int error;
    struct lwatch_path_info_t path_info;

    if ((error = database_lookup_path_info(path, &path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("database_lookup_path_info failed \"%s\" (%s)\n"),
                path, error_string(error));
        return LWATCH_ERROR_CANNOT_FIND_PATH_INFO;
    }

    path_info.flags |= NEEDS_REAPING_FLAG;

    if ((error = database_write_path_info(&path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                path_info.path, error_string(error));
        return error;
    }

    return SUCCESS;
}

/**
 * A file has been closed, test if it will need to be reaped and record reap status.
 *
 * \param[in] path Path name of the watch target to take action on.
 * \return     SUCCESS (0) or non-zero error code otherwise
 */
static int process_file_close(const char *path)
{
    int error;
    struct lwatch_path_info_t path_info;

    if ((error = database_lookup_path_info(path, &path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("database_lookup_path_info failed \"%s\" (%s)\n"),
                path, error_string(error));
        return LWATCH_ERROR_CANNOT_FIND_PATH_INFO;
    }

    if ((error = set_path_info_reap_candidacy(&path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("get_path_info_reap_candidacy failed \"%s\" (%s)\n"),
                path, error_string(error));
        return error;
    }

    if ((error = database_write_path_info(&path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                path_info.path, error_string(error));
        return error;
    }

    return SUCCESS;
}

/**
 * A file has been opened, validate it's a watch target.
 *
 * \param[in] path Path name of the watch target to take action on.
 * \return     SUCCESS (0) or non-zero error code otherwise
 */
static int process_file_create(const char *path)
{
    int error;
    struct lwatch_path_info_t path_info;

    if ((error = database_lookup_path_info(path, &path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("database_lookup_path_info failed \"%s\" (%s)\n"),
                path, error_string(error));
        return LWATCH_ERROR_CANNOT_FIND_PATH_INFO;
    }

    return SUCCESS;
}

/**
 * A file has been deleted, record that it's file contents will be different.
 *
 * \param[in] path Path name of the watch target to take action on.
 * \return     SUCCESS (0) or non-zero error code otherwise
 */
static int process_file_delete(const char *path)
{
    int error;
    struct lwatch_path_info_t path_info;

    if ((error = database_lookup_path_info(path, &path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("database_lookup_path_info failed \"%s\" (%s)\n"),
                path, error_string(error));
        return LWATCH_ERROR_CANNOT_FIND_PATH_INFO;
    }

    PATH_INFO_FLAG_SET_NEW_FILE(&path_info);

    if ((error = database_write_path_info(&path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                path_info.path, error_string(error));
        return error;
    }


    return SUCCESS;
}

/**
 * A file has been renamed, track both the old and new file contents
 *
 * \param[in] prev_path Previous path name of the watch target
 * \param[in] new_path  New path name of the watch target
 * \return     SUCCESS (0) or non-zero error code otherwise
 *
 * When a log file we've been watching is renamed we make the assumption it has
 * become a backup file (e.g. it's been rotated). It's file contents are no less
 * important to us, we must reap the file and we must still watch the file for
 * modifications. Why do we need to continue to watch a file which has been
 * renamed, doesn't the fact it's been rotated mean it won't be written to any
 * more because the file has been retired and is no longer the active log file?
 * Unfortunately this is not the case. In UNIX a file can be renamed while it is
 * open because files are identified by their inode, not their path name. Log
 * rotations are often performed by a process other than the process who has the
 * file open for writing. The process responsible for rotating files
 * (e.g. logrotate) renames the file independent of the process writing the
 * file. It may also signal the process writing the file it needs to close and
 * reopen it's files (e.g. HUP), or it may simply wait for the process writing
 * the file to close and reopen at it's convenience. All of this is
 * asynchronous, there is no way for us to know at the moment a file is renamed
 * if data will ever be added to it even though it's no longer the name of the
 * "watched log file". It is critical we do not lose any data therefore we have
 * to assume a rotated backup file has equal status with the primary file thus
 * we establish a full watch on the both the old and new versions of the file
 * being careful to record the lineage so we know how the backup file relates to
 * the primary file. This is also one of the moments when we have to record the
 * fact the primary file will need to have it's version number incremented.
 */
static int process_file_rename(const char *prev_path, const char *new_path)
{
    int error;
    struct lwatch_path_info_t prev_pathinfo, new_pathinfo;

    log_msg(LOG_DEBUG, _("\"%s\" --> \"%s\"\n"),  prev_path, new_path);

    /*
     * The backend used to watch file events may not be capable of knowing both
     * the previous and new path names thus we have to be prepared for one of
     * the two names being empty, we call this a lost rename (no rename event
     * will be generated if neither is known). Logically it should be impossible
     * for the previous name to be lost because we had a watch established on
     * it, if the previous name is lost we consider this a serious
     * error. However we might not know what the watched file was renamed to.
     * A lost new name is not considered a serious error, it's just an
     * unfortunate lack of information, but we need to be prepared for it.
     */

    if (!prev_path[0]) {
        log_msg(LOG_ERROR, _("%s: previous path is empty, should have been a file in the set of watched files\n"));
        return LWATCH_ERROR_LOST_RENAME;
    }

    if ((error = database_lookup_path_info(prev_path, &prev_pathinfo)) != SUCCESS) {
        log_msg(LOG_ERROR, _("database_lookup_path_info failed \"%s\" (%s)\n"),
                prev_path, error_string(error));
        return LWATCH_ERROR_CANNOT_FIND_PATH_INFO;
    }

    if (new_path[0]) {          /* new path is known */

        /* copy the paths, mark it as a backup, save the path_info, monitor the new file */
        new_pathinfo = prev_pathinfo;
        if ((error = copy_path(new_pathinfo.path, new_path, sizeof(new_pathinfo.path))) != SUCCESS) {
            log_msg(LOG_ERROR, _("failed to copy path \"%s\" (%s)\n"), new_path, error_string(error));
            return error;
        }
        if ((error = copy_path(new_pathinfo.original_path, prev_path, sizeof(new_pathinfo.original_path))) != SUCCESS) {
            log_msg(LOG_ERROR, _("failed to copy path \"%s\" (%s)\n"), prev_path, error_string(error));
            return error;
        }
        PATH_INFO_FLAG_SET_BACKUP(&new_pathinfo);

        if ((error = database_write_path_info(&new_pathinfo)) != SUCCESS) {
            log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                    new_pathinfo.path, error_string(error));
        }

        if ((error = inotify_start_monitoring(new_path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("could not inotify_start_monitoring for \"%s\" (%s)\n"),
                    new_path, error_string(error));
        }
    }

    /* Record the primary file will be a new file */
    prev_pathinfo.original_path[0] = 0;
    PATH_INFO_FLAG_SET_NEW_FILE(&prev_pathinfo);
    prev_pathinfo.reap_time = 0;
    prev_pathinfo.reap_position = 0;

    if ((error = database_write_path_info(&prev_pathinfo)) != SUCCESS) {
        log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                prev_pathinfo.path, error_string(error));
        return error;
    }

    return SUCCESS;
}

/**
 * Initialize our regular expressions.
 *
 * \return SUCCESS (0) or non-zero error code otherwise
 */
static int regexp_init(void)
{
    int error;

    if ((error = regexp_compile(&logrotate_numbered_backup_regexp)) != SUCCESS) return error;
    if ((error = regexp_compile(&logrotate_date_backup_regexp)) != SUCCESS) return error;
    return SUCCESS;
}

/**
 * Close our regular expression usage by releasing resources.
 *
 * \return SUCCESS (0) or non-zero error code otherwise
 */
static int regexp_fini(void)
{
    regexp_free(&logrotate_numbered_backup_regexp);
    regexp_free(&logrotate_date_backup_regexp);
    return SUCCESS;
}

/**
 * Initialize the log watcher
 *
 * \return SUCCESS (0) or non-zero error code otherwise
 *
 * - Initialize the monitoring backend
 * - Check to see what changed since the last time we ran
 */
static int lwatch_init(void)
{
    int error;

    signal(SIGHUP,  signal_handler);
    signal(SIGINT,  signal_handler);
    signal(SIGQUIT, signal_handler);

    keep_watching = true;

    /* Initialize the monitoring backend */
    if ((error = inotify_watch_init()) != SUCCESS) {
        log_msg(LOG_ERROR, _("inotify_watch_init() failed (%s)\n"),  error_string(error));
        return error;
    }
    /* Connect ourself to the backend via a callback */
    register_lwatch_event_callback(lwatch_event_callback);

    /* Check to see what changed since the last time we ran */
    if ((error = validate_backups_in_database()) != SUCCESS) {
        log_msg(LOG_ERROR, _("validate_backups_in_database() failed (%s)\n"),  error_string(error));
        return error;
    }

    return SUCCESS;
}

/**
 * Close down the log watch system
 *
 * \return SUCCESS (0) or non-zero error code otherwise
 */
static int lwatch_fini()
{
    int error;

    if ((error = inotify_watch_fini()) != SUCCESS) {
        log_msg(LOG_ERROR, _("inotify_watch_fini() failed (%s)\n"),  error_string(error));
    }

    return SUCCESS;
}

/**
 * Respond to signals
 *
 * \param[in] signum The signal number we received
 * \return void
 */
static void signal_handler (int signum)
{
    log_msg(LOG_INFO, _("received signal %s(%d)\n"), strsignal(signum), signum);

    switch(signum) {
    case SIGHUP:
        break;
    case SIGINT:
        keep_watching = false;
        break;
    case SIGQUIT:
        keep_watching = false;
        break;
    }
}

/**
 * Given a lwatch_event_type enumeration return it's name as a string
 *
 * \param[in] event_type The event type enumeration who name is desired.
 * \return Pointer to thread local static string.
 */
static const char *lwatch_event_type_string(lwatch_event_type event_type)
{
    static __thread char buf[80]; /* thread local static buffer */

    switch(event_type) {
    case LWATCH_EVENT_CREATE:      return "Created";
    case LWATCH_EVENT_DELETE:      return "Deleted";
    case LWATCH_EVENT_RENAME:      return "Renamed";
    case LWATCH_EVENT_MODIFY:      return "Modified";
    case LWATCH_EVENT_OPEN:        return "Opened";
    case LWATCH_EVENT_CLOSE:       return "Closed";
    default:
        snprintf(buf, sizeof(buf), "unknown(%d)", event_type);
        return buf;
    }
}

/**
 * Given a lwatch_event_t render it as a string
 *
 * \param[in] event The lwatch_event_t whose string representation is desired
 * \return Pointer to thread local static string.
 */
static const char *lwatch_event_string(struct lwatch_event_t *event)
{
    static __thread char buf[512 + PATH_MAX * 2]; /* thread local static buffer */
    char *p, *buf_end;

    p = buf;                     /* current position in buffer */
    buf_end = &buf[sizeof(buf)]; /* non-inclusive end of buffer */
    *p = 0;

    p += snprintf(p, buf_end - p, "[%s] ", lwatch_event_type_string(event->event_type));
    if (p >= buf_end) goto fail;
    if (event->event_type == LWATCH_EVENT_RENAME) {
        p += snprintf(p, buf_end - p, "prev_path=\"%s\" ", event->rename.prev_path);
        if (p >= buf_end) goto fail;
        p += snprintf(p, buf_end - p, "new_path=\"%s\"", event->rename.new_path);
        if (p >= buf_end) goto fail;
    } else {
        p += snprintf(p, buf_end - p, "path=\"%s\"", event->path);
        if (p >= buf_end) goto fail;
    }

    return buf;

 fail:
    /* exhausted the buffer; NULL terminate then return truncated string */
    buf_end[-1] = 0;   /* should already be NULL terminated, but be safe */
    return buf;
}

/**
 * Respond to the events sent by the monitoring back end
 *
 * \param[in] event The lwatch_event_t to process
 * \return SUCCESS (0) or non-zero error code otherwise
 */
static int lwatch_event_callback(struct lwatch_event_t *event)
{
    int error;

    log_msg(LOG_DEBUG, _("event: %s\n"),  lwatch_event_string(event));

    switch(event->event_type) {
    case LWATCH_EVENT_CREATE:
        if ((error = process_file_create(event->path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("process_file_create failed for \"%s\" (%s)\n"),
                    event->path, error_string(error));
        }
        break;
    case LWATCH_EVENT_OPEN:
        break;
    case LWATCH_EVENT_DELETE:
        if ((error = process_file_delete(event->path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("process_file_delete failed for \"%s\" (%s)\n"),
                    event->path, error_string(error));
        }
        break;
    case LWATCH_EVENT_RENAME:
        if ((error = process_file_rename(event->rename.prev_path, event->rename.new_path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("process_file_rename failed for \"%s\" --> \"%s\" (%s)\n"),
                    event->rename.prev_path, event->rename.new_path, error_string(error));
        }
        break;
    case LWATCH_EVENT_MODIFY:
        if ((error = process_file_modification(event->path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("process_file_modification failed for \"%s\" (%s)\n"),
                    event->path, error_string(error));
        }
        break;
    case LWATCH_EVENT_CLOSE:
        if ((error = process_file_close(event->path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("process_file_close failed for \"%s\" (%s)\n"),
                    event->path, error_string(error));
        }
        break;
    default:
        log_msg(LOG_ERROR, _("unknown watch event type \"%s\"\n"),  lwatch_event_string(event));
        return EINVAL;
    }

    return SUCCESS;
}

/**
 * Start monitoring a primary file
 *
 * \param[in] path_arg The path name to monitor, will be normalized and made absolute
 * \return         SUCCESS (0) or non-zero error code otherwise
 *
 * Normalize and make absolute the supplied pathname so it's in a canonical
 * form. Validate it's a primary file and not a backup of a primary file, we
 * already monitor backup's as a consequence of monitoring primary
 * files. Initialize and create an entry for this file in our persistent
 * database. Request the monitoring backend to start monitoring this file.
 */
static int start_monitoring(const char *path_arg)
{
    int error;
    struct lwatch_path_info_t path_info;
    char path[PATH_MAX];

    if ((error = make_normalized_absolute_path(path, sizeof(path), path_arg)) != SUCCESS) {
        log_msg(LOG_ERROR, _("failed make_normalized_absolute_path from \"%s\" (%s)\n"),
                path_arg, error_string(error));
        return error;
    }

    log_msg(LOG_INFO, _("monitoring \"%s\"\n"), path);

    if (is_log_backup(path)) {
        log_msg(LOG_WARN, _("file \"%s\" appears to be a log backup, only original log files should be monitored\n"),
                path);
    }

    if ((error = get_create_path_info(path, &path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("get_create_path_info failed \"%s\" (%s)\n"),
                path, error_string(error));
        return LWATCH_ERROR_CANNOT_MONITOR;
    }

    PATH_INFO_FLAG_SET_TARGET(&path_info);

    if ((error = set_path_info_reap_candidacy(&path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("set_path_info_reap_candidacy failed \"%s\" (%s)\n"),
                path, error_string(error));
        return LWATCH_ERROR_CANNOT_MONITOR;
    }

    if ((error = database_write_path_info(&path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                path_info.path, error_string(error));
        return error;
    }

    if (!is_watch_target(path)) {
        if ((error = inotify_start_monitoring(path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("could not inotify_start_monitoring for \"%s\" (%s)\n"),
                    path, error_string(error));
            return LWATCH_ERROR_CANNOT_MONITOR;
        }
    }

    return SUCCESS;
}

/**
 * Compare operating system information about a file with our saved information,
 * return how the file's state may have transformed.
 *
 * \param[in] stat_info  The stat information to compare against our saved information
 * \param[in] path_info  Our saved information about the file
 * \param[in] result_arg A file_event_t enumeration describing the state transformation
 * \return           SUCCESS (0) or non-zero error code otherwise
 *
 * After comparing the operating system information about the file to our saved
 * information returns whether the file has:
 * - not changed
 * - has been appended (data added)
 * - has been replaced by new file contents
 */
static int compare_stat_info(struct stat *stat_info, struct lwatch_path_info_t *path_info, file_event_t *result_arg)
{
    int error;
    file_event_t result;

    error = SUCCESS;
    result = FILE_NO_CHANGE;

    if (!(path_info->flags & STAT_DATA_VALID_FLAG)) {
        result = FILE_REPLACED; /* FIXME: is this really the correct state? */
        goto exit;
    }

    log_msg(LOG_DEBUG, _("st_size=%lu(%lu), st_dev=%llu(%llu) st_ino=%lu(%lu) st_mtime=%ld(%ld) \"%s\"\n"),

            stat_info->st_size,  path_info->size,
            stat_info->st_dev,   path_info->dev,
            stat_info->st_ino,   path_info->inode,
            stat_info->st_mtime, path_info->modification_time,
            path_info->path);

    /*
     * First determine if this is the same file associate with the pathname
     */

    /* If the device or inode is different it must be a different file */
    if ((stat_info->st_dev != path_info->dev) || (stat_info->st_ino != path_info->inode)) {
        result = FILE_REPLACED;
    }

    /* If the size shrunk it must be a different file since log data is appended */
    if (stat_info->st_size < path_info->size) {
        result = FILE_REPLACED;
    }

    /* If the size increased the file was appended to */
    if (stat_info->st_size > path_info->size) {
        result = FILE_APPENDED;
    }

 exit:
    *result_arg = result;
    return error;
}

/**
 * Examine the file system, set parameter if the file should be reaped.
 *
 * \param[in]   path_info  Our saved information about the file
 * \param[out] should_reap Boolean result returned here
 * \return                 SUCCESS (0) or non-zero error code otherwise
 *
 * Examine the state of the file in the file system. Compare it to our
 * information about the file. If there is information about the file or in the
 * file we need to collect (e.g. reap) then set the should_reap flag to true,
 * false otherwise.
 */
static int get_path_info_reap_candidacy(struct lwatch_path_info_t *path_info, bool *should_reap)
{
    int error;
    struct stat cur_stat_info;
    file_event_t event_type;

    *should_reap = false;

    if (stat(path_info->path, &cur_stat_info) < 0) {
        error = errno;
        if (error == ENOENT) {  /* file does not exist, not an error, just clear flag */
            return SUCCESS;
        }
        log_msg(LOG_ERROR, _("cannot perform stat on \"%s\" (%s) \n"),
                path_info->path, error_string(error));
        return error;
    }

    if ((error = compare_stat_info(&cur_stat_info, path_info, &event_type)) != SUCCESS) {
        log_msg(LOG_ERROR, _("compare_stat_info failed \"%s\" (%s) \n"),
                path_info->path, error_string(error));
        return error;
    }

    log_msg(LOG_DEBUG, _("compare_stat_info(%s) returns %s\n"),
            path_info->path, file_event_string(event_type));


    switch(event_type) {
    case FILE_NO_CHANGE:
        break;
    case FILE_APPENDED:
    case FILE_REPLACED:
        *should_reap = true;
        break;
    default:
        log_msg(LOG_ERROR, _("unknown event type = %d\n"),  event_type);
        return EINVAL;
    }

    return SUCCESS;
}

/**
 * Determine if watched file needs to be reaped, set internal flag accordingly
 *
 * \param[in] path_info Our saved information about the file, flag updated here
 * \return          SUCCESS (0) or non-zero error code otherwise
 */
static int set_path_info_reap_candidacy(struct lwatch_path_info_t *path_info)
{
    int error;
    bool should_reap;

    if ((error = get_path_info_reap_candidacy(path_info, &should_reap)) != SUCCESS) {
        if (error == ENOENT) {  /* file does not exist, not an error, just clear flag */
            path_info->flags &= ~NEEDS_REAPING_FLAG;
            return SUCCESS;
        }
    }

    if (should_reap) {
        path_info->flags |= NEEDS_REAPING_FLAG;
    } else {
        path_info->flags &= ~NEEDS_REAPING_FLAG;
    }
    return SUCCESS;
}

/**
 * Merge stat information about a file into our struct and write the updated
 * entry in the persistent database.
 *
 * \param[in] cur_stat_info Pointer to stat data to update from, or if NULL call stat()
 *                      to obtain info
 * \param[in] path_info     Our saved information about the file
 * \return              SUCCESS (0) or non-zero error code otherwise
 */
static int update_stat_info_and_write(struct stat *cur_stat_info, struct lwatch_path_info_t *path_info)
{
    int error;

    if ((error = update_stat_info(cur_stat_info, path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("could not update_stat_info for \"%s\" (%s) \n"),
                path_info->path, error_string(error));
    }

    if ((error = database_write_path_info(path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                path_info->path, error_string(error));
        return error;
    }
    return SUCCESS;
}

/*****************************************************************************/
/****************************  Exported Functions  ***************************/
/*****************************************************************************/

/**
 * Release lwatch_path_info_t resources, free it's memory, set variable to NULL.
 *
 * \param[out] ppath_info Pointer to a pointer of saved information about a file. Upon
 *                        a successful return this pointer will be set to NULL.
 * \return                SUCCESS (0) or non-zero error code otherwise
 */
int destroy_path_info(struct lwatch_path_info_t **ppath_info)
{
    struct lwatch_path_info_t *path_info;

    if (!ppath_info) return EINVAL;
    path_info = *ppath_info;
    if (!path_info) return EINVAL;

    free(path_info);
    *ppath_info = NULL;

    return SUCCESS;
}

/**
 * Merge stat information into path into our saved information about the file.
 *
 * \param[in] cur_stat_info Pointer to stat data to update from, or if NULL call stat()
 *                      to obtain info
 * \param[in] path_info     Our saved information about the file
 * \return              SUCCESS (0) or non-zero error code otherwise
 */
int update_stat_info(struct stat *cur_stat_info, struct lwatch_path_info_t *path_info)
{
    int error;
    struct stat stat_info;

    if (cur_stat_info == NULL) {
        if (stat(path_info->path, &stat_info) < 0) {
            error = errno;
            log_msg(LOG_ERROR, _("cannot perform stat on \"%s\" (%s) \n"),
                    path_info->path, error_string(error));
            return error;
        }
        cur_stat_info = &stat_info;
    }

    path_info->dev               = cur_stat_info->st_dev;
    path_info->inode             = cur_stat_info->st_ino;
    path_info->mode              = cur_stat_info->st_mode;
    path_info->uid               = cur_stat_info->st_uid;
    path_info->gid               = cur_stat_info->st_gid;
    path_info->size              = cur_stat_info->st_size;
    path_info->access_time       = cur_stat_info->st_atime;
    path_info->modification_time = cur_stat_info->st_mtime;
    path_info->change_time       = cur_stat_info->st_ctime;

    path_info->flags |= STAT_DATA_VALID_FLAG;

    return SUCCESS;
}

/**
 * Given a path name return true if it appears to be a rotated backup of a primary log file.
 *
 * \param[in] path The path name to test.
 * \return     SUCCESS (0) or non-zero error code otherwise
 *
 * By convention backing up log files is done via a process called rotation
 * where the current version of a (primary) log file is renamed to a backup name
 * and the primary log file is reopened. Usually the last N copies of the backup
 * are kept, backup.1 is moved to backup.2, backup.2 is moved to backup.3, and
 * backup.N+1 is discarded (hence the notion of rotation). Also by convention
 * the name of backup files is the name of the primary file with a rotation
 * number appended, or the date of the rotation appended.
 *
 * This routine uses regular expressions to intuit from a pathname if the
 * pathname appears to follow a rotation backup naming convention. Only the
 * pathname is considered, the file contents are \em not examined.
 *
 * Currently there are two rotation backup naming conventions which are looked
 * for, both are supported by the logrotate facility found on many machines.
 * - numbered backups
 * - dated backups
 *
 * \see logrotate_numbered_backup_regexp
 * \see logrotate_date_backup_regexp
 */
bool is_log_backup(const char *path)
{
    int num_matches;

    if ((num_matches = regexp_search(&logrotate_numbered_backup_regexp, path)) > 0) {
# if 0
        // FIXME: currently we don't use the sub-fields, this code is left as an
        // example of how to do that

        char directory[PATH_MAX];
        char filename[PATH_MAX];
        char number[9];
        char extension[PATH_MAX];

        regexp_substring(&logrotate_numbered_backup_regexp, 1, directory, sizeof(directory));
        regexp_substring(&logrotate_numbered_backup_regexp, 2, filename, sizeof(filename));
        regexp_substring(&logrotate_numbered_backup_regexp, 3, number, sizeof(number));
        regexp_substring(&logrotate_numbered_backup_regexp, 4, extension, sizeof(extension));

        if (0) printf(" num_matches=%d directory=\"%s\" filename=\"%s\" number=\"%s\" extension=\"%s\" ",
                      num_matches, directory, filename, number, extension);
#endif
        return true;
    }
    if ((num_matches = regexp_search(&logrotate_date_backup_regexp, path)) > 0) {
# if 0
        // FIXME: currently we don't use the sub-fields, this code is left as an
        // example of how to do that

        char directory[PATH_MAX];
        char filename[PATH_MAX];
        char date[9];
        char year[5];
        char month[3];
        char day[3];
        char extension[PATH_MAX];

        regexp_substring(&logrotate_date_backup_regexp, 1, directory, sizeof(directory));
        regexp_substring(&logrotate_date_backup_regexp, 2, filename, sizeof(filename));
        regexp_substring(&logrotate_date_backup_regexp, 3, date, sizeof(date));
        regexp_substring(&logrotate_date_backup_regexp, 4, year, sizeof(year));
        regexp_substring(&logrotate_date_backup_regexp, 5, month, sizeof(month));
        regexp_substring(&logrotate_date_backup_regexp, 6, day, sizeof(day));
        regexp_substring(&logrotate_date_backup_regexp, 7, extension, sizeof(extension));

        if (0) printf(" num_matches=%d directory=\"%s\" filename=\"%s\" date=\"%s\" year=\"%s\" month=\"%s\" day=\"%s\" extension=\"%s\" ",
                      num_matches, directory, filename, date, year, month, day, extension);
#endif
        return true;
    }
    return false;
}

/**
 * Display the contents of the persistent log watch database
 *
 * \return          SUCCESS (0) or non-zero error code otherwise
 *
 * Every file we've been asked to watch has state information persistently
 * stored in a database. The persistent state storage allows us to resynchronize
 * our state during intervals of time this daemon is not running without loss of
 * data. This routine will pretty print the contents of the database.
 */
static int dump_database()
{
    int error;
    struct database_iter_context_t *path_iter, *backup_iter;
    struct lwatch_path_info_t *path_info, *backup_path_info;

    if ((error = database_query_all_paths_iter(&path_iter, NULL)) != SUCCESS) {
        log_msg(LOG_ERROR, _("database_query_all_iter failed (%s)\n"), error_string(error));
    }

    while ((path_info = path_iter->next(path_iter))) {
        if (path_info->flags & BACKUP_FLAG)
            continue;

        printf("%s\n", path_info_string(path_info, ""));

        if ((error = database_query_backups_of_target_iter(&backup_iter, path_info->path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("database_query_backups_of_target_iter failed (%s)\n"), error_string(error));
        }

        while ((backup_path_info = backup_iter->next(backup_iter))) {
            printf("%s\n", path_info_string(backup_path_info, "    "));
        }
        free(backup_iter);
    }
    free(path_iter);

    return SUCCESS;
}

/**
 * Check the state of known backups since the last time we ran.
 *
 * \return          SUCCESS (0) or non-zero error code otherwise
 *
 * We need to watch every known backup file as well as the targets because a
 * backup might have data which needs to be reaped. So we iterate over all
 * backups and set a watch on each one. However a backup might have had data
 * added to it while we weren't watching (e.g. this daemon was not running at
 * the time). Thus we also need to evaluate each backup file's reap candidacy by
 * looking at it's current state in the file system and comparing it to our
 * stored information about the file (i.e. path_info). If we see the reap
 * candidacy has changed we need to update that path_info in our persistent SQL
 * storage. However we can't write records into the same table we're currently
 * iterating over. To solve this problem we could 1) call a function which
 * returns a list of path_info's, that way we would be iterating over a copy 2)
 * use a temporary table in the database to hold the result set of backup's and
 * iterate over that copy of the data, or 3) keep a local copy of what needs to
 * be updated and after the first query iteration go back and iterate over the
 * list of what needed updating and write those records. We do #3 because it's
 * the most efficient and easy to implement. First we observe the set of backup
 * files who have had data added to them and needs reaping will in most cases be
 * the empty set. So there is no point in making an extra copy of the result set
 * just in case for a small percentage of the time we need to write one of the
 * records. So instead we dynamically allocate an array to hold updates the
 * first time we see an update is needed. A dynamic array is more efficient than
 * keeping a linked list because with a linked list you have to malloc for every
 * item in the list whereas with an array you might only need to malloc once,
 * plus iterating over an array is inherently more efficient than traversing a
 * list.
 */

static int validate_backups_in_database()
{
    int error;
    struct database_iter_context_t *path_iter;
    struct lwatch_path_info_t *path_info;

    struct lwatch_path_info_t *path_info_updates;
    int i, alloc_increment, alloc_count, update_count;

    alloc_increment = 8;
    alloc_count = 0;
    update_count = 0;
    path_info_updates = NULL;

    if ((error = database_query_backups_iter(&path_iter)) != SUCCESS) {
        log_msg(LOG_ERROR, _("database_query_backups_iter failed (%s)\n"),  error_string(error));
    }

    while ((path_info = path_iter->next(path_iter))) {
        if (!(path_info->flags & NEEDS_REAPING_FLAG)) { /* no sense in checking if flag is already set */
            if ((error = set_path_info_reap_candidacy(path_info)) != SUCCESS) {
                log_msg(LOG_ERROR, _("set_path_info_reap_candidacy failed \"%s\" (%s)\n"),
                        path_info->path, error_string(error));
            }
            if (path_info->flags & NEEDS_REAPING_FLAG) { /* flag was set above, remember we have to update this path_info*/
                if (update_count == alloc_count) {
                    alloc_count += alloc_increment;
                    if ((path_info_updates = realloc(path_info_updates, alloc_count * sizeof(struct lwatch_path_info_t))) == NULL) {
                        error = ENOMEM;
                        free(path_info_updates);
                        free(path_iter);
                        log_msg(LOG_ERROR, _("could not alloc memory for update array (%s)\n"), error_string(error));
                        return error;
                    }
                }
                path_info_updates[update_count++] = *path_info;
            }
        }

        if ((error = inotify_start_monitoring(path_info->path)) != SUCCESS) {
            log_msg(LOG_ERROR, _("could not inotify_start_monitoring for \"%s\" (%s)\n"),
                    path_info->path, error_string(error));
        }
    }
    free(path_iter);

    /* If there were updates, now it's time to iterate over them and write the update */
    for (i = 0, path_info = path_info_updates; i < update_count; i++, path_info++) {
        if ((error = database_write_path_info(path_info)) != SUCCESS) {
            log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                    path_info->path, error_string(error));
        }
    }

    if (path_info_updates) free(path_info_updates);

    return SUCCESS;
}

/**
 * For each file in the database marked as needing to be reaped perform a reap.
 *
 * \return SUCCESS (0) or non-zero error code otherwise
 *
 * When we detect a file needs to be reaped we defer the actual reap operation
 * for efficiency reasons. For example if a log file is being constantly written
 * to we will get notified upon each write operation. We don't want to reap
 * every individual write to the file, rather it's more efficient to wait an
 * interval of time (\see reap_interval) and collect all the new data and
 * transmit it in a single operation. This is much more foregiving of system
 * resources.
 */
static int process_pending_reaps()
{
    int error;
    struct database_iter_context_t *iter;
    struct lwatch_path_info_t *path_info;

    log_msg(LOG_DEBUG, _("process_pending_reaps\n"));

    if ((error = database_query_pending_reaps_iter(&iter)) != SUCCESS) {
        log_msg(LOG_ERROR, _("database_query_pending_reaps_iter failed (%s)\n"),
                error_string(error));
        return error;
    }

    while ((path_info = iter->next(iter))) {
        if ((error = reap_file_data(path_info)) != SUCCESS) {
            log_msg(LOG_ERROR, _("cannot reap file data for \"%s\" (%s)\n"),
                    path_info->path, error_string(error));
        }
    }
    free(iter);

    return SUCCESS;
}

/**
 * The file has already been identified for reaping, perform the reap by
 * collecting the file contents and updating our bookkeeping.
 *
 * \param[in] path_info  Our saved information about the file
 * \return          SUCCESS (0) or non-zero error code otherwise
 */
static int reap_file_data(struct lwatch_path_info_t *path_info)
{
    int error;
    struct stat stat_info;
    const char *path;
    size_t length;

    path = path_info->path;
    if (stat(path, &stat_info) < 0) {
        error = errno;
        log_msg(LOG_ERROR, _("cannot perform stat on \"%s\" (%s) \n"),
                path, error_string(error));

        if (error == ENOENT) {  /* path does not exist, report error, remove reap request */
            int write_error;

            log_msg(LOG_ERROR, _("deleted \"%s\" before being fully collected\n"), path);

            path_info->flags &= ~NEEDS_REAPING_FLAG;

            if ((write_error = database_write_path_info(path_info)) != SUCCESS) {
                log_msg(LOG_ERROR, _("could not insert path_info into database for \"%s\" (%s)\n"),
                        path_info->path, error_string(write_error));
            }
        }
        return error;
    }

    if (path_info->flags & INC_VERSION_FLAG) {
        path_info->version++;
        path_info->flags &= ~INC_VERSION_FLAG;
    }

    length = stat_info.st_size - path_info->reap_position;

    printf("REAP \"%s\" [%lu:%lu]\n", path, path_info->reap_position, path_info->reap_position + length);

    path_info->reap_position = path_info->reap_position + length;
    path_info->reap_time = time(NULL);
    path_info->flags &= ~NEEDS_REAPING_FLAG;
    if ((error = update_stat_info_and_write(&stat_info, path_info)) != SUCCESS) {
        log_msg(LOG_ERROR, _("could not update_stat_info_and_write for \"%s\" (%s) \n"),
                path, error_string(error));
        return error;
    }

    return SUCCESS;
}

/**
 * Main loop when running in daemon monitor mode
 *
 * \return SUCCESS (0) or non-zero error code otherwise
 *
 * Enter a loop which keeps processing events until the keep_watching global
 * flag is set to false. The action of the event loop is:
 * - read event from inotify backend and process it.
 * - if there are no events to process sleep until the reap_interval expires
 *   then wake up and process any pending reaps which were queued.
 */
static int main_loop()
{
    int error;
    int result;
    fd_set read_fds;
    int inotify_fd;
    struct timeval timeout;

    inotify_fd = get_inotify_fd();

    timeout.tv_sec = reap_interval;
    timeout.tv_usec = 0;

    while (keep_watching) {
        FD_ZERO(&read_fds);
        FD_SET(inotify_fd, &read_fds);

        result = select(inotify_fd + 1, &read_fds, NULL, NULL, &timeout);
        if (result == -1) {     /* error */
            error = errno;
            if (error != EINTR) {
                log_msg(LOG_ERROR, _("select error (%s)\n"),
                        error_string(error));
            }
        } else if (result) {    /* data available */
            read_events();
        } else {                /* timeout */
            /* The timeout expired, it needs to be reset */
            timeout.tv_sec = reap_interval;
            timeout.tv_usec = 0;

            if ((error = process_pending_reaps()) != SUCCESS) {
                log_msg(LOG_ERROR, _("error processing pending reaps (%s)\n"),
                        error_string(error));
            }
        }
    }

    return SUCCESS;
}

/**
 * Recursive directory walk callback used to find potential log files
 *
 * \param[in] directory The current directory during the walk
 * \param[in] filename  The basename of the current directory entry
 * \param[in] path      The full pathname of the current directory entry
 *                  (e.g. directory/filename)
 * \param[in] info      stat struct providing information about the directory entry
 * \param[in] user_data Pointer to a count of found files
 * \return          True to keep walking
 *
 * This callback is invoked for every directory entry found while walking a
 * directory tree. We want to locate potential log files and report them to the
 * user. To be considered a log file the directory entry must:
 * - must be a regular file or a link
 * - not be a log backup (based on the path name)
 * otherwise the file is reported as a potential log file
 */
static bool find_log_files_callback(const char *directory, const char *filename, const char *path, struct stat *info, void *user_data)
{
    int *count = (int *)user_data;

    if (!(S_ISREG(info->st_mode) || S_ISLNK(info->st_mode))) return true;

    if (is_log_backup(path)) return true;

    printf("%s\n", path);
    (*count)++;

    return true;
}

/**
 * Find potential log files in a directory (and below)
 *
 * \param[in] dir_path  Directory to search (from)
 * \param[in] recursive True if the search should recursively descend into other directories.
 * \return          SUCCESS (0) or non-zero error code otherwise
 *
 * Starting at the specified directory and emit the full pathname of every file
 * found which has the potential of being a primary log file. If recursive is
 * true then descend into sub-directories as well. The test is performed by
 * examing the pathname of the file. Files which appear to be backups of a
 * primary log file (i.e. the result of log file rotation) are ignored.
 * \see find_log_files_callback()
 */
static int find_log_files(const char *dir_path, int recursive)
{
    int error;
    char path[PATH_MAX];
    int count;

    if ((error = make_normalized_absolute_path(path, sizeof(path), dir_path)) != SUCCESS) {
        log_msg(LOG_ERROR, _("failed make_normalized_absolute_path from \"%s\" (%s)\n"),
                dir_path, error_string(error));
        return error;
    }

    count = 0;
    directory_list(path, recursive, find_log_files_callback, &count);

    return SUCCESS;
}

/*****************************************************************************/
/***********************************  Main  **********************************/
/*****************************************************************************/

/**
 * Write the program usage help information
 */
static void usage(void)
{
    const char *msg = "\
-h --help                 provide usage information\n\
-f --find dir1 [dir2 ...] find all log files in dir(s)\n\
-r --recursive            when combined with --find examine all subdirectories too\n\
-l --list                 dump the contents of the watch database\n\
-i --interval seconds     number of seconds in reap period\n\
   --log_level level      set logging level, one of:\n\
                          error, warn, info, debug, verbose_debug\n\
   --log_to_console       logging messsage appear on stdout\n\
   --logfile_path         pathname of the log file\n\
   --no_logging_to_file   disable logging to file\n\
   --log_show_level       show logging level in log message\n\
   --log_no_show_level    do not show logging level in log message\n\
   --log_show_file        show file name in log message\n\
   --log_no_show_file     do not show file name in log message\n\
   --log_show_line        show line number in log message\n\
   --log_no_show_line     do not show line number in log message\n\
   --log_show_function    show function name in log message\n\
   --log_no_show_function do not show function name in log message\n\
   --log_show_time        show time stamp in log message\n\
   --log_no_show_time     do not show time stamp in log message\n\
";
    printf(msg);
}

/**
 * Main program entry point.
 *
 * \param[in] argc Count of how many program arguments
 * \param[in] argv Array of program argument strings
 * \return     Exit code passed back to invoking environment.
 *
 * Behavior is:
 * - Initialize logging
 * - Process program arguments
 * - Determine which operation mode to run in
 *   - monitor
 *   - find
 *   - list
 * - Initialize sub-systems
 * - Execute in requested mode
 * - Close sub-systems
 * - Exit
 */
int main(int argc, char **argv)
{
    int error;
    int i;
    prog_action_t prog_action = PROG_ACTION_MONITOR;
    bool recursive = false;
    int log_level = LOG_WARN;
    char *logfile_path = NULL;

    log_set_show_level(true);
    log_set_show_file(false);
    log_set_show_line(false);
    log_set_show_function(true);
    log_set_show_time(false);


    while (1) {
        int arg;
        int option_index = 0;
        static struct option long_options[] = {
            {"help",                 0, 0, 'h'},
            {"find",                 0, 0, 'f'},
            {"recursive",            0, 0, 'r'},
            {"list",                 0, 0, 'l'},
            {"interval",             1, 0, 'i'},
            {"log_level",            1, 0, LOG_LEVEL_ARG},
            {"log_to_console",       0, 0, LOG_TO_CONSOLE_ARG},
            {"logfile_path",         0, 0, LOGFILE_PATH_ARG},
            {"no_logging_to_file",   0, 0, NO_LOGGING_TO_FILE_ARG},
            {"log_show_level",       0, 0, LOG_SHOW_LEVEL_ARG},
            {"log_no_show_level",    0, 0, LOG_NO_SHOW_LEVEL_ARG},
            {"log_show_file",        0, 0, LOG_SHOW_FILE_ARG},
            {"log_no_show_file",     0, 0, LOG_NO_SHOW_FILE_ARG},
            {"log_show_line",        0, 0, LOG_SHOW_LINE_ARG},
            {"log_no_show_line",     0, 0, LOG_NO_SHOW_LINE_ARG},
            {"log_show_function",    0, 0, LOG_SHOW_FUNCTION_ARG},
            {"log_no_show_function", 0, 0, LOG_NO_SHOW_FUNCTION_ARG},
            {"log_show_time",        0, 0, LOG_SHOW_TIME_ARG},
            {"log_no_show_time",     0, 0, LOG_NO_SHOW_TIME_ARG},
            {0, 0, 0, 0}
        };

        arg = getopt_long(argc, argv, "hfrli:",
                          long_options, &option_index);
        if (arg == -1) break;

        switch (arg) {
        case 'h':
            usage();
            exit(0);
            break;
        case 'f':
            prog_action = PROG_ACTION_FIND;
            break;
        case 'r':
            recursive = true;
            break;
        case 'l':
            prog_action = PROG_ACTION_LIST;
            break;
        case 'i':
            reap_interval = atoi(optarg);
            break;
        case LOG_LEVEL_ARG:
            if (log_level_from_string(optarg, &log_level) != SUCCESS) {
                fprintf(stderr, _("unknown log level: %s\n"), optarg);
            }
            break;
        case LOG_TO_CONSOLE_ARG:
            log_to_console(true);
            break;
        case LOGFILE_PATH_ARG:
            logfile_path = optarg;
            break;
        case NO_LOGGING_TO_FILE_ARG:
            log_to_file(false);
            break;

        case LOG_SHOW_LEVEL_ARG:
            log_set_show_level(true);
            break;
        case LOG_NO_SHOW_LEVEL_ARG:
            log_set_show_level(false);
            break;
        case LOG_SHOW_FILE_ARG:
            log_set_show_file(true);
            break;
        case LOG_NO_SHOW_FILE_ARG:
            log_set_show_file(false);
            break;
        case LOG_SHOW_LINE_ARG:
            log_set_show_line(true);
            break;
        case LOG_NO_SHOW_LINE_ARG:
            log_set_show_line(false);
            break;
        case LOG_SHOW_FUNCTION_ARG:
            log_set_show_function(true);
            break;
        case LOG_NO_SHOW_FUNCTION_ARG:
            log_set_show_function(false);
            break;
        case LOG_SHOW_TIME_ARG:
            log_set_show_time(true);
            break;
        case LOG_NO_SHOW_TIME_ARG:
            log_set_show_time(false);
            break;

        default:
            usage();
            exit(1);
            break;
        }
    }

    log_set_filepath(logfile_path);
    log_set_level(log_level);
    if ((error = log_init()) != SUCCESS) {
        fprintf(stderr, _("log initialization failed (%s)\n"), error_string(error));
        return 1;
    }

    if ((error = regexp_init()) != SUCCESS) {
        log_msg(LOG_ERROR, _("regexp initialization failed (%s)\n"), error_string(error));
        return 1;
    }

    if ((error = watch_database_init()) != SUCCESS) {
        log_msg(LOG_ERROR, _("watch_database initialization failed (%s)\n"), error_string(error));
        return 1;
    }

    if ((error = lwatch_init()) != SUCCESS) {
        log_msg(LOG_ERROR, _("lwatch initialization failed (%s)\n"), error_string(error));
        return 1;
    }

    switch (prog_action) {
    case PROG_ACTION_MONITOR:
        for (i = optind; i < argc; i++) {
            char *path = argv[i];

            if ((error = start_monitoring(path)) != SUCCESS) {
                log_msg(LOG_ERROR, _("could not monitor \"%s\" (%s)\n"),
                        path, error_string(error));
            }
        }

        main_loop();

        /* Before we exit assure we reap any outstanding data which might still be pending */
        if ((error = process_pending_reaps()) != SUCCESS) {
            log_msg(LOG_ERROR, _("error processing pending reaps (%s)\n"),
                    error_string(error));
        }

        break;
    case PROG_ACTION_FIND:
        for (i = optind; i < argc; i++) {
            char *path = argv[i];

            if ((error = find_log_files(path, recursive)) != SUCCESS) {
                log_msg(LOG_ERROR, _("could not find log files \"%s\" (%s)\n"),
                        path, error_string(error));
            }
        }
        break;
    case PROG_ACTION_LIST:
        dump_database();
        break;
    }

    if ((error = lwatch_fini()) != SUCCESS) {
        log_msg(LOG_ERROR, _("lwatch finalization failed (%s)\n"), error_string(error));
    }

    if ((error = regexp_fini()) != SUCCESS) {
        log_msg(LOG_ERROR, _("regexp finalization failed (%s)\n"), error_string(error));
    }

    if ((error = watch_database_fini()) != SUCCESS) {
        log_msg(LOG_ERROR, _("watch_database finalization failed (%s)\n"), error_string(error));
    }

    if ((error = log_fini()) != SUCCESS) {
        fprintf(stderr, _("log finalization failed (%s)\n"), error_string(error));
    }

    return 0;
}

---