Xcode/LogFileCompressor/CompressingLogFileManager.m

#import "CompressingLogFileManager.h"
#import <zlib.h>

// We probably shouldn't be using DDLog() statements within the DDLog implementation.
// But we still want to leave our log statements for any future debugging,
// and to allow other developers to trace the implementation (which is a great learning tool).
// 
// So we use primitive logging macros around NSLog.
// We maintain the NS prefix on the macros to be explicit about the fact that we're using NSLog.

#define LOG_LEVEL 4

#define NSLogError(frmt, ...)    do{ if(LOG_LEVEL >= 1) NSLog(frmt, ##__VA_ARGS__); } while(0)
#define NSLogWarn(frmt, ...)     do{ if(LOG_LEVEL >= 2) NSLog(frmt, ##__VA_ARGS__); } while(0)
#define NSLogInfo(frmt, ...)     do{ if(LOG_LEVEL >= 3) NSLog(frmt, ##__VA_ARGS__); } while(0)
#define NSLogVerbose(frmt, ...)  do{ if(LOG_LEVEL >= 4) NSLog(frmt, ##__VA_ARGS__); } while(0)

@interface CompressingLogFileManager (/* Must be nameless for properties */)

@property (readwrite) BOOL isCompressing;

@end

@interface DDLogFileInfo (Compressor)

@property (nonatomic, readonly) BOOL isCompressed;

- (NSString *)tempFilePathByAppendingPathExtension:(NSString *)newExt;
- (NSString *)fileNameByAppendingPathExtension:(NSString *)newExt;

@end

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#pragma mark -
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

@implementation CompressingLogFileManager

@synthesize isCompressing;

- (id)init
{
    return [self initWithLogsDirectory:nil];
}

- (id)initWithLogsDirectory:(NSString *)aLogsDirectory
{
    if ((self = [super initWithLogsDirectory:aLogsDirectory]))
    {
        upToDate = NO;
        
        // Check for any files that need to be compressed.
        // But don't start right away.
        // Wait for the app startup process to finish.
        
        [self performSelector:@selector(compressNextLogFile) withObject:nil afterDelay:5.0];
    }
    return self;
}

- (void)dealloc
{
    [NSObject cancelPreviousPerformRequestsWithTarget:self selector:@selector(compressNextLogFile) object:nil];
}

- (void)compressLogFile:(DDLogFileInfo *)logFile
{
    self.isCompressing = YES;

    CompressingLogFileManager* __weak weakSelf = self;
    dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0), ^{
        [weakSelf backgroundThread_CompressLogFile:logFile];
    });
}

- (void)compressNextLogFile
{
    if (self.isCompressing)
    {
        // We're already compressing a file.
        // Wait until it's done to move onto the next file.
        return;
    }
    
    NSLogVerbose(@"CompressingLogFileManager: compressNextLogFile");
    
    upToDate = NO;
    
    NSArray *sortedLogFileInfos = [self sortedLogFileInfos];
    
    NSUInteger count = [sortedLogFileInfos count];
    if (count == 0)
    {
        // Nothing to compress
        upToDate = YES;
        return;
    }
    
    NSUInteger i = count;
    while (i > 0)
    {
        DDLogFileInfo *logFileInfo = [sortedLogFileInfos objectAtIndex:(i - 1)];
        
        if (logFileInfo.isArchived && !logFileInfo.isCompressed)
        {
            [self compressLogFile:logFileInfo];
            
            break;
        }
        
        i--;
    }
    
    upToDate = YES;
}

- (void)compressionDidSucceed:(DDLogFileInfo *)logFile
{
    NSLogVerbose(@"CompressingLogFileManager: compressionDidSucceed: %@", logFile.fileName);
    
    self.isCompressing = NO;
    
    [self compressNextLogFile];
}

- (void)compressionDidFail:(DDLogFileInfo *)logFile
{
    NSLogWarn(@"CompressingLogFileManager: compressionDidFail: %@", logFile.fileName);
    
    self.isCompressing = NO;
    
    // We should try the compression again, but after a short delay.
    // 
    // If the compression failed there is probably some filesystem issue,
    // so flooding it with compression attempts is only going to make things worse.
    
    NSTimeInterval delay = (60 * 15); // 15 minutes
    
    [self performSelector:@selector(compressNextLogFile) withObject:nil afterDelay:delay];
}

- (void)didArchiveLogFile:(NSString *)logFilePath
{
    NSLogVerbose(@"CompressingLogFileManager: didArchiveLogFile: %@", [logFilePath lastPathComponent]);
    
    // If all other log files have been compressed,
    // then we can get started right away.
    // Otherwise we should just wait for the current compression process to finish.
    
    if (upToDate)
    {
        [self compressLogFile:[DDLogFileInfo logFileWithPath:logFilePath]];
    }
}

- (void)didRollAndArchiveLogFile:(NSString *)logFilePath
{
    NSLogVerbose(@"CompressingLogFileManager: didRollAndArchiveLogFile: %@", [logFilePath lastPathComponent]);
    
    // If all other log files have been compressed,
    // then we can get started right away.
    // Otherwise we should just wait for the current compression process to finish.
    
    if (upToDate)
    {
        [self compressLogFile:[DDLogFileInfo logFileWithPath:logFilePath]];
    }
}

- (void)backgroundThread_CompressLogFile:(DDLogFileInfo *)logFile
{
    @autoreleasepool {
    
    NSLogInfo(@"CompressingLogFileManager: Compressing log file: %@", logFile.fileName);
    
    // Steps:
    //  1. Create a new file with the same fileName, but added "gzip" extension
    //  2. Open the new file for writing (output file)
    //  3. Open the given file for reading (input file)
    //  4. Setup zlib for gzip compression
    //  5. Read a chunk of the given file
    //  6. Compress the chunk
    //  7. Write the compressed chunk to the output file
    //  8. Repeat steps 5 - 7 until the input file is exhausted
    //  9. Close input and output file
    // 10. Teardown zlib
    
    
    // STEP 1
    
    NSString *inputFilePath = logFile.filePath;
    
    NSString *tempOutputFilePath = [logFile tempFilePathByAppendingPathExtension:@"gz"];
    
#if TARGET_OS_IPHONE
    // We use the same protection as the original file.  This means that it has the same security characteristics.
    // Also, if the app can run in the background, this means that it gets
    // NSFileProtectionCompleteUntilFirstUserAuthentication so that we can do this compression even with the
    // device locked.  c.f. DDFileLogger.doesAppRunInBackground.
    NSString* protection = logFile.fileAttributes[NSFileProtectionKey];
    NSDictionary* attributes = protection == nil ? nil : @{NSFileProtectionKey: protection};
    [[NSFileManager defaultManager] createFileAtPath:tempOutputFilePath contents:nil attributes:attributes];
#endif
    
    // STEP 2 & 3
    
    NSInputStream *inputStream = [NSInputStream inputStreamWithFileAtPath:inputFilePath];
    NSOutputStream *outputStream = [NSOutputStream outputStreamToFileAtPath:tempOutputFilePath append:NO];
    
    [inputStream open];
    [outputStream open];
    
    // STEP 4
    
    z_stream strm;
    
    // Zero out the structure before (to be safe) before we start using it
    bzero(&strm, sizeof(strm));
    
    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    strm.total_out = 0;
    
    // Compresssion Levels:
    //   Z_NO_COMPRESSION
    //   Z_BEST_SPEED
    //   Z_BEST_COMPRESSION
    //   Z_DEFAULT_COMPRESSION
    
    deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, (15+16), 8, Z_DEFAULT_STRATEGY);
    
    // Prepare our variables for steps 5-7
    // 
    // inputDataLength  : Total length of buffer that we will read file data into
    // outputDataLength : Total length of buffer that zlib will output compressed bytes into
    // 
    // Note: The output buffer can be smaller than the input buffer because the
    //       compressed/output data is smaller than the file/input data (obviously).
    // 
    // inputDataSize : The number of bytes in the input buffer that have valid data to be compressed.
    // 
    // Imagine compressing a tiny file that is actually smaller than our inputDataLength.
    // In this case only a portion of the input buffer would have valid file data.
    // The inputDataSize helps represent the portion of the buffer that is valid.
    // 
    // Imagine compressing a huge file, but consider what happens when we get to the very end of the file.
    // The last read will likely only fill a portion of the input buffer.
    // The inputDataSize helps represent the portion of the buffer that is valid.
    
    NSUInteger inputDataLength  = (1024 * 2);  // 2 KB
    NSUInteger outputDataLength = (1024 * 1);  // 1 KB
    
    NSMutableData *inputData = [NSMutableData dataWithLength:inputDataLength];
    NSMutableData *outputData = [NSMutableData dataWithLength:outputDataLength];
    
    NSUInteger inputDataSize = 0;
    
    BOOL done = YES;
    NSError* error = nil;
    do
    {
        @autoreleasepool {
        
        // STEP 5
        // Read data from the input stream into our input buffer.
        // 
        // inputBuffer : pointer to where we want the input stream to copy bytes into
        // inputBufferLength : max number of bytes the input stream should read
        // 
        // Recall that inputDataSize is the number of valid bytes that already exist in the
        // input buffer that still need to be compressed.
        // This value is usually zero, but may be larger if a previous iteration of the loop
        // was unable to compress all the bytes in the input buffer.
        // 
        // For example, imagine that we ready 2K worth of data from the file in the last loop iteration,
        // but when we asked zlib to compress it all, zlib was only able to compress 1.5K of it.
        // We would still have 0.5K leftover that still needs to be compressed.
        // We want to make sure not to skip this important data.
        // 
        // The [inputData mutableBytes] gives us a pointer to the beginning of the underlying buffer.
        // When we add inputDataSize we get to the proper offset within the buffer
        // at which our input stream can start copying bytes into without overwriting anything it shouldn't.
        
        const void *inputBuffer = [inputData mutableBytes] + inputDataSize;
        NSUInteger inputBufferLength = inputDataLength - inputDataSize;
        
        NSInteger readLength = [inputStream read:(uint8_t *)inputBuffer maxLength:inputBufferLength];
        if (readLength < 0) {
            error = [inputStream streamError];
            break;
        }
        
        NSLogVerbose(@"CompressingLogFileManager: Read %li bytes from file", (long)readLength);
        
        inputDataSize += readLength;
        
        // STEP 6
        // Ask zlib to compress our input buffer.
        // Tell it to put the compressed bytes into our output buffer.
        
        strm.next_in = (Bytef *)[inputData mutableBytes];   // Read from input buffer
        strm.avail_in = (uInt)inputDataSize;                // as much as was read from file (plus leftovers).
        
        strm.next_out = (Bytef *)[outputData mutableBytes]; // Write data to output buffer
        strm.avail_out = (uInt)outputDataLength;            // as much space as is available in the buffer.
        
        // When we tell zlib to compress our data,
        // it won't directly tell us how much data was processed.
        // Instead it keeps a running total of the number of bytes it has processed.
        // In other words, every iteration from the loop it increments its total values.
        // So to figure out how much data was processed in this iteration,
        // we fetch the totals before we ask it to compress data,
        // and then afterwards we subtract from the new totals.
        
        NSInteger prevTotalIn = strm.total_in;
        NSInteger prevTotalOut = strm.total_out;
        
        int flush = [inputStream hasBytesAvailable] ? Z_SYNC_FLUSH : Z_FINISH;
        deflate(&strm, flush);
        
        NSInteger inputProcessed = strm.total_in - prevTotalIn;
        NSInteger outputProcessed = strm.total_out - prevTotalOut;
        
        NSLogVerbose(@"CompressingLogFileManager: Total bytes uncompressed: %lu", (unsigned long)strm.total_in);
        NSLogVerbose(@"CompressingLogFileManager: Total bytes compressed: %lu", (unsigned long)strm.total_out);
        NSLogVerbose(@"CompressingLogFileManager: Compression ratio: %.1f%%",
                     (1.0F - (float)(strm.total_out) / (float)(strm.total_in)) * 100);
        
        // STEP 7
        // Now write all compressed bytes to our output stream.
        // 
        // It is theoretically possible that the write operation doesn't write everything we ask it to.
        // Although this is highly unlikely, we take precautions.
        // Also, we watch out for any errors (maybe the disk is full).
        
        NSUInteger totalWriteLength = 0;
        NSInteger writeLength = 0;
        
        do
        {
            const void *outputBuffer = [outputData mutableBytes] + totalWriteLength;
            NSUInteger outputBufferLength = outputProcessed - totalWriteLength;
            
            writeLength = [outputStream write:(const uint8_t *)outputBuffer maxLength:outputBufferLength];
            
            if (writeLength < 0)
            {
                error = [outputStream streamError];
            }
            else
            {
                totalWriteLength += writeLength;
            }
            
        } while((totalWriteLength < outputProcessed) && !error);
        
        // STEP 7.5
        // 
        // We now have data in our input buffer that has already been compressed.
        // We want to remove all the processed data from the input buffer,
        // and we want to move any unprocessed data to the beginning of the buffer.
        // 
        // If the amount processed is less than the valid buffer size, we have leftovers.
        
        NSUInteger inputRemaining = inputDataSize - inputProcessed;
        if (inputRemaining > 0)
        {
            void *inputDst = [inputData mutableBytes];
            void *inputSrc = [inputData mutableBytes] + inputProcessed;
            
            memmove(inputDst, inputSrc, inputRemaining);
        }
        
        inputDataSize = inputRemaining;
        
        // Are we done yet?
        
        done = ((flush == Z_FINISH) && (inputDataSize == 0));
        
        // STEP 8
        // Loop repeats until end of data (or unlikely error)
        
        } // end @autoreleasepool
        
    } while (!done && error == nil);
    
    // STEP 9
    
    [inputStream close];
    [outputStream close];
    
    // STEP 10
    
    deflateEnd(&strm);
    
    // We're done!
    // Report success or failure back to the logging thread/queue.
    
    if (error)
    {
        // Remove output file.
        // Our compression attempt failed.

        NSLogError(@"Compression of %@ failed: %@", inputFilePath, error);
        error = nil;
        BOOL ok = [[NSFileManager defaultManager] removeItemAtPath:tempOutputFilePath error:&error];
        if (!ok)
            NSLogError(@"Failed to clean up %@ after failed compression: %@", tempOutputFilePath, error);
        
        // Report failure to class via logging thread/queue
        
        dispatch_async([DDLog loggingQueue], ^{ @autoreleasepool {
            
            [self compressionDidFail:logFile];
        }});
    }
    else
    {
        // Remove original input file.
        // It will be replaced with the new compressed version.

        error = nil;
        BOOL ok = [[NSFileManager defaultManager] removeItemAtPath:inputFilePath error:&error];
        if (!ok)
            NSLogWarn(@"Warning: failed to remove original file %@ after compression: %@", inputFilePath, error);
        
        // Mark the compressed file as archived,
        // and then move it into its final destination.
        // 
        // temp-log-ABC123.txt.gz -> log-ABC123.txt.gz
        // 
        // The reason we were using the "temp-" prefix was so the file would not be
        // considered a log file while it was only partially complete.
        // Only files that begin with "log-" are considered log files.
        
        DDLogFileInfo *compressedLogFile = [DDLogFileInfo logFileWithPath:tempOutputFilePath];
        compressedLogFile.isArchived = YES;
        
        NSString *outputFileName = [logFile fileNameByAppendingPathExtension:@"gz"];
        [compressedLogFile renameFile:outputFileName];
        
        // Report success to class via logging thread/queue
        
        dispatch_async([DDLog loggingQueue], ^{ @autoreleasepool {
            
            [self compressionDidSucceed:compressedLogFile];
        }});
    }
    
    } // end @autoreleasepool
}
                 
@end

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#pragma mark -
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

@implementation DDLogFileInfo (Compressor)

@dynamic isCompressed;

- (BOOL)isCompressed
{
    return [[[self fileName] pathExtension] isEqualToString:@"gz"];
}

- (NSString *)tempFilePathByAppendingPathExtension:(NSString *)newExt
{
    // Example:
    // 
    // Current File Name: "/full/path/to/log-ABC123.txt"
    // 
    // newExt: "gzip"
    // result: "/full/path/to/temp-log-ABC123.txt.gzip"
    
    NSString *tempFileName = [NSString stringWithFormat:@"temp-%@", [self fileName]];
    
    NSString *newFileName = [tempFileName stringByAppendingPathExtension:newExt];
    
    NSString *fileDir = [[self filePath] stringByDeletingLastPathComponent];
    
    NSString *newFilePath = [fileDir stringByAppendingPathComponent:newFileName];
    
    return newFilePath;
}


- (NSString *)fileNameByAppendingPathExtension:(NSString *)newExt
{
    // Example:
    // 
    // Current File Name: "log-ABC123.txt"
    // 
    // newExt: "gzip"
    // result: "log-ABC123.txt.gzip"
    
    NSString *fileNameExtension = [[self fileName] pathExtension];
    
    if ([fileNameExtension isEqualToString:newExt])
    {
        return [self fileName];
    }
    
    return [[self fileName] stringByAppendingPathExtension:newExt];
}

@end