This section of the archives stores flipcode's complete Developer Toolbox collection, featuring a variety of mini-articles and source code contributions from our readers.

 

  Multi-Threaded Logging Library
  Submitted by



This is a high performance, multi-threaded logging library which I've made, which some people may find useful. The main features are:

  • Multiple threads and *multiple processes* can access the same log file at the same time. That means if you have a number of related programs, they can all use the same log file without problems.
  • The logger automatically prepends useful information to each line before it outputs it, things like a timestamp, process name and thread ID.
  • The logger doesn't let the log file get too big. Once it hits 256KB, it makes a backup and starts with a fresh file. This is useful because if you try and open a huge file in notepad, it can take days :) Also, it makes it easy to find entries that happened weeks or months ago (since the backups include the current date/time in the filename)
  • High performance. Actual writing to the file is done in a separate thread, so you don't have to wait for the I/O to complete before logging another line (it'll get added to a queue if you try and log a line and it's already writing a line) or continuing on with your program.


  • If you like this library, the don't forget to head on over to my home page:
    http://www.codeka.com - it's a little empty now, but I'll be adding to it as I go. I'll post any updates to this library there.

    Dean Harding.

    Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/internal.h] - (2,074 bytes)

    //-----------------------------------------------------------------------------
    // INTERNAL.H
    // By Dean Harding
    // 
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    #pragma once

    //----------------------------------------------------------------------------- #include <windows.h> #include <tchar.h> #include <stdio.h> #include <stdlib.h> #include <assert.h>

    #ifdef _DEBUG #define ASSERT(x) assert(x) #else #define ASSERT(x) (x) #endif

    //----------------------------------------------------------------------------- #include "logger.h" #include "item.h" #include "queue.h"

    //----------------------------------------------------------------------------- // worker thread that does all the actual logging work. See worker.cpp for // it's implementation. DWORD WINAPI LoggerWorkerThread( void *pData );

    //----------------------------------------------------------------------------- // This is the main queue that all the loggers add to and the worker thread // reads from. I know, I know, it's a global variable - so kill me! extern CQueue<CLoggerItem>g_LoggerQueue;

    //----------------------------------------------------------------------------- // this is the class that actually implements the functionality of CLogger. // see loggerimpl.cpp for it's implementation. class CLoggerImpl : public CLogger { protected: // current format, as set by SetFormat() DWORD m_dwFormat;

    // current process name, as set by SetName() TCHAR *m_szProcName;

    // current log file name, as set by SetLogFile() TCHAR *m_szLogFile; public: CLoggerImpl( TCHAR *szLogFile ); ~CLoggerImpl();

    void Release();

    void SetFormat( DWORD dwFormat ); DWORD GetFormat();

    void SetName( TCHAR *szName ); TCHAR *GetName();

    void Print( TCHAR *szFormat, ... ); };

    //----------------------------------------------------------------------------- // End of INTERNAL.H //-----------------------------------------------------------------------------

    Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/item.h] - (1,210 bytes)

    //-----------------------------------------------------------------------------
    // ITEM.H
    // By Dean Harding
    // 
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    // This represents one item to be written to the log file.
    //-----------------------------------------------------------------------------
    #pragma once

    //----------------------------------------------------------------------------- // Note, anything not specified by dwFormat is ignored. class CLoggerItem { public: TCHAR *szLine; TCHAR *szLogFile; TCHAR *szProcName; DWORD dwThreadId; FILETIME ftTimestamp;

    DWORD dwFormat;

    CLoggerItem( DWORD dwFormat, TCHAR *szLine, TCHAR *szLogFile, TCHAR *szProcName, DWORD dwThreadId, FILETIME ftTimestamp ) : dwFormat( dwFormat ), szLine( szLine ), szLogFile( szLogFile ) , szProcName( szProcName ), dwThreadId( dwThreadId ) , ftTimestamp( ftTimestamp ) {}

    ~CLoggerItem() { delete[] szLine; } };

    //----------------------------------------------------------------------------- // End of ITEM.H //-----------------------------------------------------------------------------

    Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/logger.cpp] - (1,458 bytes)

    //-----------------------------------------------------------------------------
    // LOGGER.CPP
    // By Dean Harding
    // 
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    #include "internal.h"

    HANDLE g_hThread = NULL;

    //----------------------------------------------------------------------------- void CLogger::Initialize() { if( g_hThread != NULL ) { // initialize has already been called. return; }

    DWORD dwThreadId;

    // create the thread that will actually do the logging g_hThread = CreateThread( NULL, 0, LoggerWorkerThread, NULL, 0, &dwThreadId );

    }

    //----------------------------------------------------------------------------- void CLogger::Shutdown() { if( g_hThread == NULL ) { // Initialize hasn't been called, or Shutdown has already been called. return; }

    // add a NULL item to the queue. That the thread's queue to quit. g_LoggerQueue.AddItem( NULL );

    WaitForSingleObject( g_hThread, INFINITE ); CloseHandle( g_hThread );

    g_hThread = NULL; }

    //----------------------------------------------------------------------------- CLogger *CLogger::Attach( TCHAR *szLogFile ) { return new CLoggerImpl( szLogFile ); }

    //----------------------------------------------------------------------------- // End of LOGGER.CPP //-----------------------------------------------------------------------------

    Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/logger.h] - (2,481 bytes)

    //-----------------------------------------------------------------------------
    // LOGGER.H
    // By Dean Harding
    // 
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    #pragma once

    #include <tchar.h>

    //----------------------------------------------------------------------------- #ifdef LOGGER_EXPORTS #define LOGGER_API __declspec(dllexport) #else #define LOGGER_API __declspec(dllimport) #endif

    //----------------------------------------------------------------------------- // The logger can print a number of things along with the actual line it's // logging. These are what it can also print #define LOG_TIMESTAMP 1 // the time the line was logged #define LOG_PROCNAME 2 // the name of the process that logged the // line. This can be set with // CLogger::SetName() #define LOG_THREADID 4 // the thread ID that logged this line //----------------------------------------------------------------------------- // Maximum length of a line #define LOG_MAX_LINE 256

    //----------------------------------------------------------------------------- class LOGGER_API CLogger { public: // Initialize the logger. This must be called before any calls to Attach() // (and it must have completed before any calls to Attach as well. static void Initialize();

    // Shutdown the logger. Any loggers that have been created will become // invalid! static void Shutdown();

    // Attach to the logger. This just creates a new CLogger class for you to // use. static CLogger *Attach( TCHAR *szLogFile = _T("errors.log") );

    // When you're finished logging, call this to destroy the CLogger class. virtual void Release() = 0;

    // Sets/Gets the current things that are printed along with actual line. virtual void SetFormat( DWORD dwFormat ) = 0; virtual DWORD GetFormat() = 0;

    // set/get the name displayed by LOG_PROCNAME. If NULL is given, then we // use the name of the executable this process is running from (eg // myapp.exe) virtual void SetName( TCHAR *szName ) = 0; virtual TCHAR *GetName() = 0;

    // this is the big one! This actually writes a line to the log file. virtual void Print( TCHAR *szFormat, ... ) = 0; };

    //----------------------------------------------------------------------------- // End of LOGGER.H //-----------------------------------------------------------------------------

    Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/loggerimpl.cpp] - (2,944 bytes)

    //-----------------------------------------------------------------------------
    // LOGGERIMPL.CPP
    // By Dean Harding
    // 
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    #include "internal.h"

    //----------------------------------------------------------------------------- CLoggerImpl::CLoggerImpl( TCHAR *szLogFile ) : m_szLogFile( NULL ), m_szProcName( NULL ), m_dwFormat( 0 ) { m_szLogFile = new TCHAR[_tcslen( szLogFile ) + 1]; _tcscpy( m_szLogFile, szLogFile );

    SetName( NULL ); } //----------------------------------------------------------------------------- CLoggerImpl::~CLoggerImpl() { delete[] m_szProcName; delete[] m_szLogFile; }

    //----------------------------------------------------------------------------- void CLoggerImpl::Release() { // wait for the queue to empty. That's the only way we know that none of // our data members are still being used. g_LoggerQueue.WaitForEmpty();

    delete this; }

    //----------------------------------------------------------------------------- void CLoggerImpl::SetFormat( DWORD dwFormat ) { m_dwFormat = dwFormat; } //----------------------------------------------------------------------------- DWORD CLoggerImpl::GetFormat() { return m_dwFormat; }

    //----------------------------------------------------------------------------- void CLoggerImpl::SetName( TCHAR *szName ) { if( m_szProcName != NULL ) { delete[] m_szProcName; }

    if( szName != NULL ) { m_szProcName = new TCHAR[_tcslen(szName) + 1]; _tcscpy( m_szProcName, szName ); } else { TCHAR szBuffer[MAX_PATH]; GetModuleFileName( NULL, szBuffer, MAX_PATH ); TCHAR *p = (TCHAR *)(szBuffer + _tcslen(szBuffer)); while( *(p - 1) != '\\' && p != szBuffer ) p--;

    m_szProcName = new char[_tcslen(p) + 1]; _tcscpy( m_szProcName, p ); } } //----------------------------------------------------------------------------- TCHAR *CLoggerImpl::GetName() { return m_szProcName; }

    //----------------------------------------------------------------------------- void CLoggerImpl::Print( TCHAR *szFormat, ... ) { FILETIME ftTimestamp = { 0 }; DWORD dwThreadId = 0; TCHAR *szBuffer = new TCHAR[LOG_MAX_LINE];

    va_list args; va_start( args, szFormat ); _vstprintf( szBuffer, szFormat, args ); va_end( args );

    if( (m_dwFormat & LOG_TIMESTAMP) != 0 ) { GetSystemTimeAsFileTime( &ftTimestamp ); }

    if( (m_dwFormat & LOG_THREADID) != 0 ) { dwThreadId = GetCurrentThreadId(); }

    CLoggerItem *pItem = new CLoggerItem( m_dwFormat, szBuffer, m_szLogFile, m_szProcName, dwThreadId, ftTimestamp ); g_LoggerQueue.AddItem( pItem ); }

    //----------------------------------------------------------------------------- // End of LOGGERIMPL.CPP //-----------------------------------------------------------------------------

    Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/queue.h] - (2,613 bytes)

    //-----------------------------------------------------------------------------
    // QUEUE.H
    // By Dean Harding
    // 
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    // I needed a fast queue system, where it's very fast *adding* to the *end* of
    // the queue, and *removing* from the beginning (i.e. FIFO).  I don't know of
    // any STL container that have those properties, so I made my own.
    //
    // It's just a doubly-linked list :)
    //
    // Also, this class is thread safe (a thread can add to the queue while another
    // is getting stuff from the top)  And finally, you can block on a GetNext()
    // call until data is added.  Note that this only works when there is one
    // *and only one* thread using GetNext().  That's OK - that's exactly the
    // functionality we want!
    //-----------------------------------------------------------------------------
    #pragma once

    //----------------------------------------------------------------------------- template< class T > class CQueue { protected: struct SQueueData { T *pData;

    SQueueData *pNext; SQueueData *pPrev; };

    SQueueData *m_pTop; SQueueData *m_pBottom;

    // We just lock the whole queue when we want to add/remove to/from it, // because that's nice and easy, and besides, the things we do on the queue //should be simple enough that it won't really matter. HANDLE m_hQueueLock;

    // When we add an item queue, this even it fired to tell anything waiting // in GetNext(). HANDLE m_hAddEvent;

    // This is used by WaitForEmpty(). When GetNext() removes the last item // from the queue, this is signalled. HANDLE m_hEmptyEvent; public: inline CQueue(); inline ~CQueue();

    // Get the first item off the top of the queue. If the queue is empty // and bBlock is false, NULL is returned. If the queue is empty and bBlock // is true, then the function blocks until the queue is added to. inline T* GetNext( bool bBlock = false );

    // Add an item to the queue. If there's a blocking call to GetNext() then // that call will wake up and return this item. inline void AddItem( T* pItem );

    // returns true if the queue is empty. inline bool IsEmpty();

    // blocks until the queue is empty inline void WaitForEmpty(); };

    //----------------------------------------------------------------------------- #include "queue.inl"

    //----------------------------------------------------------------------------- // End of QUEUE.H //-----------------------------------------------------------------------------

    Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/worker.cpp] - (5,959 bytes)

    //-----------------------------------------------------------------------------
    // WORKER.CPP
    // By Dean Harding
    // 
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    // This is the thread that actually does all the work.  It waits on the
    // queue for more items, whenever a new item is added, it writes that item
    // to a file!
    //-----------------------------------------------------------------------------
    #include "internal.h"

    //----------------------------------------------------------------------------- CQueue<CLoggerItem>g_LoggerQueue;

    //----------------------------------------------------------------------------- // While this lock is set, no threads from other processes will try and open // the file. This is for things like when renaming a large file or such HANDLE g_hFileLock = NULL; const TCHAR g_szFileLock[] = _T("dhLoggerFileLock");

    //----------------------------------------------------------------------------- // renames an open file, and returns a handle to a new file of the same name. HANDLE RenameFile( HANDLE hFile, TCHAR *szFileName );

    //----------------------------------------------------------------------------- // opens a file. If access is denied, keep trying HANDLE OpenFile( TCHAR *szFileName );

    //----------------------------------------------------------------------------- DWORD WINAPI LoggerWorkerThread( void *pData ) { TCHAR szBuffer[1024]; DWORD dwNumWritten;

    g_hFileLock = CreateMutex( NULL, FALSE, g_szFileLock ); while( true ) { CLoggerItem *pItem = g_LoggerQueue.GetNext( true );

    // if it's a "NULL" item, then that's our queue to quit! if( pItem == NULL ) break;

    HANDLE hFile = OpenFile( pItem->szLogFile );

    DWORD dwFileSize = GetFileSize( hFile, NULL ); if( dwFileSize > (256 * 1024) ) { // if the file is bigger than 256KB, then rename it and make a new // one hFile = RenameFile( hFile, pItem->szLogFile ); }

    // seek to the end of the file SetFilePointer( hFile, 0, 0, FILE_END );

    // if they want a timestamp, format that for them if( (pItem->dwFormat & LOG_TIMESTAMP) != 0 ) { FILETIME ftLocalTime; SYSTEMTIME stLocalTime;

    FileTimeToLocalFileTime( &pItem->ftTimestamp, &ftLocalTime ); FileTimeToSystemTime( &ftLocalTime, &stLocalTime );

    _stprintf( szBuffer, _T("%2d/%02d/%04d %2d:%02d:%02d.%03d "), stLocalTime.wDay, stLocalTime.wMonth, stLocalTime.wYear, stLocalTime.wHour, stLocalTime.wMinute, stLocalTime.wSecond, stLocalTime.wMilliseconds );

    WriteFile( hFile, szBuffer, _tcslen( szBuffer ), &dwNumWritten, NULL ); }

    // if they want a process name & thread id if( (pItem->dwFormat & LOG_PROCNAME) != 0 && (pItem->dwFormat & LOG_THREADID) != 0 ) { _stprintf( szBuffer, _T("(%s:%d) "), pItem->szProcName, pItem->dwThreadId );

    WriteFile( hFile, szBuffer, _tcslen( szBuffer ), &dwNumWritten, NULL ); } // if it's just the process name else if( (pItem->dwFormat & LOG_PROCNAME) != 0 ) { _stprintf( szBuffer, _T("(%s)"), pItem->szProcName );

    WriteFile( hFile, szBuffer, _tcslen( szBuffer ), &dwNumWritten, NULL ); } // if it's just the thread id else if( (pItem->dwFormat & LOG_THREADID) != 0 ) { _stprintf( szBuffer, _T("(%d)"), pItem->dwThreadId );

    WriteFile( hFile, szBuffer, _tcslen( szBuffer ), &dwNumWritten, NULL ); }

    // now write the actual line WriteFile( hFile, pItem->szLine, _tcslen( pItem->szLine ), &dwNumWritten, NULL );

    // and an end-of-line WriteFile( hFile, _T("\r\n"), _tcslen( _T("\r\n") ), &dwNumWritten, NULL );

    // and close the file! CloseHandle( hFile );

    delete pItem; }

    CloseHandle( g_hFileLock );

    return 0; }

    //----------------------------------------------------------------------------- HANDLE RenameFile( HANDLE hFile, TCHAR *szFileName ) { ASSERT( WaitForSingleObject( g_hFileLock, INFINITE ) == WAIT_OBJECT_0 );

    // now that we own the file lock, we can close the file, rename and make a // new one!! CloseHandle( hFile );

    TCHAR szBuffer[MAX_PATH]; TCHAR szNewFile[MAX_PATH]; FILETIME ftCurrent, ftLocal; SYSTEMTIME stLocal;

    TCHAR *p = (TCHAR *)(szFileName + _tcslen(szFileName) ); while( *(p - 1) != '\\' && p != szFileName ) p--;

    GetSystemTimeAsFileTime( &ftCurrent ); FileTimeToLocalFileTime( &ftCurrent, &ftLocal ); FileTimeToSystemTime( &ftLocal, &stLocal );

    _stprintf( szBuffer, _T("%04d-%02d-%2d %02d-%02d-%02d %s"), stLocal.wYear, stLocal.wMonth, stLocal.wDay, stLocal.wHour, stLocal.wMinute, stLocal.wSecond, p );

    _tcsncpy( szNewFile, szFileName, (p - szFileName) ); p = szNewFile + (p - szFileName); *p = '\0';

    _tcscat( szNewFile, szBuffer );

    MoveFile( szFileName, szNewFile );

    ReleaseMutex( g_hFileLock ); return OpenFile( szFileName ); }

    //----------------------------------------------------------------------------- HANDLE OpenFile( TCHAR *szFileName ) { // open the file, keep trying until it's not INVALID_HANDLE_VALUE // There has to be a better way than this. I mean, I want to be able // to block the thread until the file is available again. I don't know // if there's any API for that... HANDLE hFile = INVALID_HANDLE_VALUE; while( hFile == INVALID_HANDLE_VALUE ) { // we only try to open the file if we own the hFileLock mutex. ASSERT( WaitForSingleObject( g_hFileLock, INFINITE ) == WAIT_OBJECT_0 );

    hFile = CreateFile( szFileName, GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL ); Sleep( 0 );

    ReleaseMutex( g_hFileLock ); }

    return hFile; }

    //----------------------------------------------------------------------------- // End of WORKER.CPP //-----------------------------------------------------------------------------

    Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger_test/logger_test.cpp] - (1,762 bytes)

    //-----------------------------------------------------------------------------
    // LOGGER_TEST.CPP
    // By Dean Harding
    // 
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    // This is just a little console application which creates a number of threads
    // and writes stuff to the log file.  It's just to test logger.dll
    //-----------------------------------------------------------------------------
    #include <windows.h>
    #include <stdio.h>
    #include <conio.h>
    #include <iostream.h>
    #include "..\logger\logger.h"

    //----------------------------------------------------------------------------- int main( int argc, char **argv ) { cout << "Hello World!!" << endl;

    CLogger::Initialize();

    CLogger *logger = CLogger::Attach( _T("log\\errors.log") ); if( logger == NULL ) { cout << "Could not attack to logger." << endl; return 1; }

    logger->SetFormat( LOG_PROCNAME | LOG_TIMESTAMP | LOG_THREADID ); DWORD dwProcessId = GetCurrentProcessId(); DWORD dwRun = 1; while( !kbhit() ) { cout << "Process: " << dwProcessId << ", Run: " << dwRun << endl; logger->Print( "Process: %d, Run: %d", dwProcessId, dwRun ); dwRun ++;

    // make it so the log file doesn't get too big, too fast... // if you remove this, then expect a large lot of log files :) //Sleep( 100 ); }

    cout << "Flushing log file..." << endl; logger->Release(); logger = NULL;

    cout << "Finishing logging..." << endl; CLogger::Shutdown();

    getch();

    return 0; }

    //----------------------------------------------------------------------------- // End of LOGGER_TEST.CPP //-----------------------------------------------------------------------------

    The zip file viewer built into the Developer Toolbox made use of the zlib library, as well as the zlibdll source additions.

     

    Copyright 1999-2008 (C) FLIPCODE.COM and/or the original content author(s). All rights reserved.
    Please read our Terms, Conditions, and Privacy information.