/*

Papers plugin to import from the ACM website
(c) 2007.

Released into the public domain by Niko Matsakis with absolutely no warranty.

*/

#import "importer.h"
#import "OAMacros.h"
#import "OACategories.h"
#import "OAEncoding.h"

#define ERROR_DOMAIN             @"Papers Importer Error"
#define ERROR_NO_URL             0
#define ERROR_NO_URL_TEXT        NSLocalizedString(@"NoClipboardURL", @"No url found in the clipboard")
#define ERROR_BAD_ENCODING       1
#define ERROR_BAD_ENCODING_TEXT  NSLocalizedString(@"BadEncoding", @"Unexpected encoding which cannot be converted to UTF-8")
#define ERROR_NO_BIBTEX          2
#define ERROR_NO_BIBTEX_TEXT     NSLocalizedString(@"NoBibtex", @"Could not find link to bibtex file")

#ifndef NDEBUG
#  define DebugLog(args...) NSLog(args)
#else
#  define DebugLog(args...)
#endif

@interface mtAcmNdmImporter (private)

- (void) processURL:(NSURL*)url;

@end


@implementation mtAcmNdmImporter

#pragma mark - 
#pragma mark Init

- (id) init {
    self = [super init];
	if ( self != nil ) {	
        DebugLog(@"Initialized mtAcmNdmImporter");
        // This part is HIGHLY dependent on the current ACM pages!
        int opts = REG_EXTENDED | REG_ICASE;
        r_bibtex_url = [[OARegex regexWithPattern:@"popBibTex.cfm\\?id=[^']*" options:opts] retain];
        r_shorter_url = [[OARegex regexWithPattern:@"http://portal.acm.org/citation.cfm\\?id=[0-9]*" options:opts] retain];

        // Build up the r_bibtexs array:
        NSMutableArray *tmp = [NSMutableArray array];
        NSArray *fieldFormats = [NSArray arrayWithObjects:
            @"([a-zA-Z_]*) *= *{([^}]*)}",     // key = {value}
            @"([a-zA-Z_]*) *= *\"([^\"]*)\"",  // key = "value"
            nil];
        FOREACH(NSString *, fieldFormat, fieldFormats) {
            NSString *bibTexFormat = [NSString stringWithFormat:
                @"@ *[a-zA-Z_]* *{[[:space:]]*"           // @article {
                @"([a-zA-Z0-9:_-]*,)?[[:space:]]*"        //    123456,
                @"(%@,?[[:space:]]*)*"                    //    key = {value}
                @"}", fieldFormat];                       // }
            NSArray *pair = [NSArray arrayWithObjects:
                [OARegex regexWithPattern:bibTexFormat options:opts],
                [OARegex regexWithPattern:fieldFormat options:opts],
                nil];
            [tmp addObject:pair]; 
        } ENDFOREACH;
        r_bibtexs = [[NSArray arrayWithArray:tmp] retain]; // store an immutable copy
    }
	return self;    
}

- (void) awakeFromNib {
	// setup nib if necessary
}

- (void) dealloc {
    // cleanup last items here
    [r_bibtex_url release];
    [r_bibtexs release];
    [urls release];
    [keywords release];
    [super dealloc];
}


#pragma mark -
#pragma mark Accessors

// these are pretty standard accessors and already take care of most signalling of progress
// to the delegate, usually not much need for changes here

- (id)delegate
{
	return delegate;
}

- (void)setDelegate:(id)newDelegate
{
	delegate = newDelegate;
}


- (NSNumber *)itemsToImport
{
	return itemsToImport;
}

- (void)setItemsToImport:(NSNumber *)newItemsToImport
{
	[newItemsToImport retain];
	[itemsToImport release];
	itemsToImport = newItemsToImport;
	
	// inform our delegate to update the status
	id <PapersImporterPluginDelegate> del = [self delegate];
	[del updateStatus: self];
}


- (NSNumber *)importedItems
{
	return importedItems;
}

- (void)setImportedItems:(NSNumber *)newImportedItems
{
	[newImportedItems retain];
	[importedItems release];
	importedItems = newImportedItems;
	
	// inform our delegate to update the status
	id <PapersImporterPluginDelegate> del = [self delegate];
	[del updateStatus: self];
}

- (void)incrementImportedItemsWith: (int)value{
	int old = [[self importedItems]intValue];
	[self setImportedItems: [NSNumber numberWithInt: old+value]];
}


- (NSString *)statusString
{
	return statusString;
}

- (void)setStatusString:(NSString *)newStatusString
{
	[newStatusString retain];
	[statusString release];
	statusString = newStatusString;
	
	// inform our delegate to update the status
	id <PapersImporterPluginDelegate> del = [self delegate];
	[del updateStatus: self];
}


- (NSError *)importError
{
	return importError;
}

- (void)setImportError:(NSError *)newImportError
{
	[newImportError retain];
	[importError release];
	importError = newImportError;
}

- (void)setImportError:(int)code text:(NSString*)text
{
    NSError *error = [NSError errorWithDomain:ERROR_DOMAIN code:code userInfo:
        [NSDictionary dictionaryWithObjectsAndKeys:NSLocalizedDescriptionKey, text, nil]];
    [self setImportError:error];
}


#pragma mark -
#pragma mark Interface interaction methods

// this should return an array of all menu items as strings.  Menu titles should be as descriptive as possible,
// so that we don't have naming collisions.
- (NSArray *) menuTitles{
	return [NSArray arrayWithObjects: NSLocalizedString(@"ACM Import", @"Name of first menu title"), nil];
}


#pragma mark -
#pragma mark Open Panel methods

// You can open a sheet, interact with a website, fetch data etc. Just return NO here in that case.
- (BOOL) shouldShowOpenPanel{
	return NO;
}

// if your plugin returns YES on the above method, you have to return the allowedFileTypes as well.
- (NSArray *) allowedFileTypes{
	return nil; //[NSArray arrayWithObjects: @"my_extension", @"my_other_extension", nil];
}

// indicate what can be selected. Return NO when you don't open a file.
- (BOOL) canChooseFiles{
	return NO;
}

- (BOOL) canChooseDirectories{
	return NO;
}

- (BOOL) requiresInternetConnection{
	return YES;
}

- (BOOL) canChooseMultiple{
	return NO;
}

- (BOOL) canCancelImport{
	return YES;
}

// if you would like to customize the openpanel even further, set one up and return your own. 
// Otherwise return nil for the default one.
- (NSOpenPanel *) openPanel{
	return nil;
}

// if you would like to show options while the open panel is shown, return a custom view here and it will
// be used as accessoryView. Otherwise return nil.
- (NSView *) accessoryView{
	return nil;
}

#pragma mark -
#pragma mark Preparation methods

// a method to make sure everything's set to go before starting, do some setup or tests here if necessary.
// and a method to find out what the problems are if things aren't set. See above for usage of errorCodes.
- (BOOL) readyToPerformImport{	
	// do some setup here if necessary
    canceled = NO;
	[self setImportedItems: [NSNumber numberWithInt: 0]];
	return YES;
}

- (NSError *) importPreparationError{
	return importError;
}

// Return YES if imported results should be linked to existing objects as much as possible (like authors). Returning NO might 
// result in duplicates
- (BOOL)shouldMerge{
	return YES;
}

// Return YES if imported results should overwrite existing fields during a merge. For instance you are importing an author already 
// present in the library with a different email address.
- (BOOL)shouldOverwrite {
	return NO;    // for now, don't overwrite, as metadata we extract is rather shallow.
}

#pragma mark -
#pragma mark Asking the users for the URLs

- (void) askUser {
    NSApplication *app = [NSApplication sharedApplication];
    
    // first, fetch any URLs from the pasteboard
    NSPasteboard *pb = [NSPasteboard generalPasteboard];
    NSString *urlStr = [pb stringForType:NSStringPboardType];
    if (urlStr != nil) {
        NSURL *url = [[NSURL alloc] initWithString:urlStr];
        if (url != nil) {
            // valid URL
            [urlsTextView setString:[url description]];
            [url release];
        }
    }
    
    // now, let the user edit them and (if the user does not cancel)
    // set the url/keyword arrays
    int result = [app runModalForWindow:accessoryWindow];
    [accessoryWindow close];
    if (result == NSRunStoppedResponse) {
        urls = [[[urlsTextView string] componentsSeparatedByWhitespace] retain];
        NSString *allKeywords = [[keywordsTextField stringValue] normalizeWhitespace];
        if (![allKeywords isEqualToString:@""])
            keywords = [[[allKeywords componentsSeparatedByString:@","] mapArrayWith:@selector(normalizeWhitespace)] retain];
    }
    else {
        // aborted
        urls = keywords = nil;
    }
}

- (IBAction) import:sender {
    NSApplication *app = [NSApplication sharedApplication];
    [app stopModal];
}

- (IBAction) dontImport:sender {
    NSApplication *app = [NSApplication sharedApplication];
    [app abortModal];
}

#pragma mark -
#pragma mark Import methods

// this method is the main worker method and launches the import process, here you are handed over
// the URLs that were selected in the open panel, this array can be empty or nil if you return NO
// to shouldShowOpenPanel, in case you want to ignore this argument anyway.
// NOTE that this method runs in a separate THREAD. Signal your progress to the delegate.
- (void) performImportWithURLs: (NSArray *)unused {
	// Since we run threaded we need the autorelease pool!
	NSAutoreleasePool *pool = [[NSAutoreleasePool alloc]init];
    
    // Start by asking the user for the URL.  Sets the fields 'urls' and
    // 'keywords' when finished either to nil or valid objects based on user input.
    [self performSelectorOnMainThread:@selector(askUser) withObject:nil waitUntilDone:YES];
    
    // Tell the delegate we began importing.
    id <PapersImporterPluginDelegate> del = [self delegate];    
    [del didBeginImport: self];
    
    if (urls) {
        [self setItemsToImport:[NSNumber numberWithInt:[urls count]]];
        for (int i = 0, ic = [urls count]; i < ic && !canceled; i++) {
            [self setImportedItems:[NSNumber numberWithInt:i+1]];
            NSString *urlText = [urls objectAtIndex:i];
            NSURL *url = [[[NSURL alloc] initWithString:urlText] autorelease];
            if (url != nil) {
                [self processURL:url];
            }
        }
        
    }
    
    // done, let the delegate know
    if ([del respondsToSelector:@selector(didEndImport:)])
        [del didEndImport:self]; // XXX should be didEndImport:self, but seems to be a bug in the tester
    
	// cleanup nicely
	[pool release];
}

- (NSString*) normalizeAccents:(NSString*)bibtext
{
    // Some of the accents seem to fit a pattern:
    char *texAccents = "`'^~\"";
    struct PatternCase { char c; int base; } patterns[] = 
    { 
    { 'A', 192 }, { 'a', 224 },
    { 'E', 200 }, { 'e', 232 },
    { 'I', 204 }, { 'i', 236 },
    { 'O', 210 }, { 'o', 242 },
    { 'U', 217 }, { 'u', 249 },
    { 0, 0 }
    };
    
    // Special case replacements:
    struct SpecialCase {
        NSString *seq; unichar replace; 
    } specials[] = {
    { @"\\c{c}", 231 },
    { @"\\c{C}", 199 },
    { @"\\~{n}", 241 },
    { @"\\~{N}", 209 },        
    { @"\\'{Y}", 221 },        
    { @"\\'{y}", 253 },        
    { @"\\\"{y}", 255 },
    { nil, 0 }
    };
    
    // Make the required replacements:
    NSMutableString *res = [NSMutableString stringWithString:bibtext];
    
    //    Special cases:
    for (struct SpecialCase *c = specials; c->seq != nil; c++) {
        NSString *rep = [NSString stringWithCharacters:&c->replace length:1];
        [res replaceOccurrencesOfString:c->seq withString:rep options:0 range:NSMakeRange(0, [res length])];
    }
    
    //    Pattern cases:
    for (struct PatternCase *p = patterns; p->c != 0; p++) {
        for (int a = 0; a < strlen(texAccents); a++) {
            NSString *tex1 = [NSString stringWithFormat:@"\\%c{%c}", texAccents[a], p->c];
            NSString *tex2 = [NSString stringWithFormat:@"\\%c{\\%c}", texAccents[a], p->c];
            unichar rep_char = p->base+a;
            NSString *rep = [NSString stringWithCharacters:&rep_char length:1];
            [res replaceOccurrencesOfString:tex1 withString:rep options:0 range:NSMakeRange(0, [res length])];
            [res replaceOccurrencesOfString:tex2 withString:rep options:0 range:NSMakeRange(0, [res length])];
        }
    }
    
    return res;    
}

// Given one "name=value" pair from the bibtex, edits the paper dictionary appropriately.
- (void) processField:(NSString*)fld 
                value:(NSString*)value 
            paperDict:(NSMutableDictionary*)paperDict
          journalDict:(NSMutableDictionary*)journalDict
{
    if ([fld isEqualToString:@"author"]) {
        // Handle the author field very specially.
        NSMutableArray *authorDicts = [NSMutableArray array];
        NSArray *authorStrings = [value componentsSeparatedByString:@" and "];
        FOREACH(NSString *, author, authorStrings) {
            // Papers wants the following fields: first name, initials, last name.
            NSArray *words = [author componentsSeparatedByString:@" "];
            if ([words count] == 0)
                continue; // ignore empty strings...
                
            // Extract first name and initials.
            NSString *firstName = nil;
            NSMutableString *initials = [NSMutableString string];        
            for (int i = 0; i < [words count] - 1; i++) {
                NSString *word = [words objectAtIndex:i];
                if (firstName == nil) firstName = word;
                [initials appendFormat:@"%C", [word characterAtIndex:0]];
            }
            
            // Make the dictionary that represents an author.
            NSMutableDictionary *authorDict = [NSMutableDictionary dictionary];
            if (firstName != nil) [authorDict setObject:firstName forKey:@"firstName"];
            [authorDict setObject:initials forKey:@"initials"];
            [authorDict setObject:[words lastObject] forKey:@"lastName"];
            
            [authorDicts addObject:authorDict];
        } ENDFOREACH;
        [paperDict setObject:authorDicts forKey:@"authors"];
    }
    else if ([fld isEqualToString:@"year"]) {
        NSNumber *num = [NSNumber numberWithInt:[value intValue]];
        [paperDict setObject:num forKey:@"year"];
    }
    else if ([fld isEqualToString:@"journal"])
        [journalDict setObject:value forKey:@"name"];
    else if ([fld isEqualToString:@"number"])
        [paperDict setObject:value forKey:@"issue"];
        
    // Hack: apply some proceeding fields to journals:
    //    (but add a note about it)
    else if ([fld isEqualToString:@"booktitle"]) {
        [journalDict setObject:value forKey:@"name"];
        [journalDict setObject:@"(Proceedings)" forKey:@"notes"];
    }
    else if ([fld isEqualToString:@"isbn"])
        [journalDict setObject:value forKey:@"issn"];
        
    else {
        NSArray *include = [NSArray arrayWithObjects:@"doi", @"title", @"pages", @"volume", nil];
        if ([include containsObject:fld])
            [paperDict setObject:value forKey:fld];
            
        NSArray *includeJ = [NSArray arrayWithObjects:@"issn", @"publisher", nil];
        if ([includeJ containsObject:fld])
            [journalDict setObject:value forKey:fld];
    }
}

- (NSString*) fetchURL:(NSURL*)url
{
    // Fetch the contents of the URL.  Unfortunately, 
    // [NSString stringWithContentsOfURL:] does not seem to work.
    NSURLResponse *resp = nil;
    NSError *error = nil;
    NSURLRequest *req = [NSURLRequest requestWithURL:url];
    NSData *data = [NSURLConnection sendSynchronousRequest:req returningResponse:&resp error:&error];
    if (data == nil) {
        NSLog(@"Papers ACM Importer: Error fetching contents of %@: %@", url, error);
        [self setImportError:error];
        return nil;
    }

    // Find the encoding that they used, and convert to utf-8.
    // If they didn't specify, or there is some error, just 
    // try to use the bytes they gave us as is (here we assume that we
    // mostly care about ASCII content, which is usually the same
    // in all encodings!)
    NSString *textEncodingName = [resp textEncodingName];        
    OAEncoding *oaencoding = [OAEncoding encodingForName:textEncodingName];
    NSStringEncoding enc = (oaencoding == nil ? NSUTF8StringEncoding : [oaencoding encoding]);
    DebugLog(@"Converting from encoding %@ to UTF-8", textEncodingName);
    NSString *dataString = [[[NSString alloc] initWithData:data encoding:[oaencoding encoding]] autorelease];
    return dataString;
}

- (NSString*) extractAbstractFrom:(NSString*)mainText
{
    // The abstract in the ACM stuff consists of many entries like:
    //
    //   <p class="abstract">...</p>
    //   ...
    //   <p class="abstract">...</p>
    //
    // Unfortunately, they use class=abstract for many things, so we must add some
    // hacky code to exclude stuff we don't want.    
    
    NSMutableArray *paragraphTexts = [NSMutableArray array];
    
    // Search through the abstract division for abstract paragraphs:
    NSString *paragraphTag = @"<p class=\"abstract\">";
    NSRange remainingString = [mainText rangeAfter:0];
    for (;;) {
        NSRange paragraphStart = [mainText rangeOfString:paragraphTag
                                                 options:NSCaseInsensitiveSearch 
                                                   range:remainingString];
        if (paragraphStart.location == NSNotFound)
            break;
        
        // Find the next "</p>" from the start of the paragraph.
        NSRange afterParagraphStart = [mainText rangeAfterRange:paragraphStart];
        NSRange paragraphEnd = [mainText rangeOfString:@"</p>"
                                               options:NSCaseInsensitiveSearch
                                                 range:afterParagraphStart];
        if (paragraphEnd.location == NSNotFound)  // found no </p>... just bail out I guess.
            break;
        
        // Extract what lies between.
        NSRange paragraphContents = {
            .location = afterParagraphStart.location,
            .length = paragraphEnd.location - afterParagraphStart.location };
        NSString *paragraphText = [mainText substringWithRange:paragraphContents];
        if (![paragraphText containsString:@"<font color=\"Red\">Note: OCR errors"])
            [paragraphTexts addObject:[[paragraphText stripHTML] normalizeWhitespace]];
        
        // Adjust the remaining range.
        remainingString = [mainText rangeAfterRange:paragraphEnd];
    }
    
    if ([paragraphTexts count] != 0) {
        // Combine the paragraphs with \n and set the abstract.
        NSString *abstract = [NSString stringByJoining:paragraphTexts withString:@"\n"];
        return abstract;
    }
    return nil;
}

- (void) processURL:(NSURL*)mainUrl
{
    DebugLog(@"Processing URL: %@", mainUrl);

    NSString *abstract = nil;
    NSString *bibText = nil;
    
    /* ----------------------------------------------------------------- */
    // ACM Web pages:
    //     here, we scrape the abstract from the main page
    //     and locate a link to the bibtex
    if ([[mainUrl host] containsString:@"acm.org"]) {
        // Fetch text of the main page.
        NSString *mainText;
        if (!(mainText = [self fetchURL:mainUrl])) {
            [self setImportError:ERROR_NO_BIBTEX text:ERROR_NO_BIBTEX_TEXT];
            return;
        }
        
        // Extract the abstract (if we find any) from the web page
        abstract = [self extractAbstractFrom:mainText];
        
        // Find the Bibtex URL.
        //   I used to make it relative to mainUrl, but if mainUrl is the DOI
        //   URL, this doesn't work.
        NSString *bibUrlString = [r_bibtex_url matchedSubstringOfString:mainText];
        if (bibUrlString == nil) {
            [self setImportError:ERROR_NO_BIBTEX text:ERROR_NO_BIBTEX_TEXT];
            return;
        }
        NSURL *bibUrl = [NSURL URLWithString:bibUrlString 
                               relativeToURL:[NSURL URLWithString:@"http://portal.acm.org/"]];
        
        // Fetch bibtex.
        bibText = [self fetchURL:bibUrl];
    }
    
    /* ----------------------------------------------------------------- */
    // All other web pages:
    //     Here we assume the bibtex is inline in the text of the page
    //     (as it often is)
    else {
        bibText = [self fetchURL:mainUrl];
    }
    
    
    /* ----------------------------------------------------------------- */
    // Generic bib-tex processing
    //    We iterate through the bibText string, finding all citations
    //    and processing them one-by-one.  This processing is mostly shared 
    //    between all kinds of pages.

    if (bibText == nil) {
        [self setImportError:ERROR_NO_BIBTEX text:ERROR_NO_BIBTEX_TEXT];
        return;
    } 
    
    // Preprocess bibtex to remove accented characters which mess up my regex engine.
    bibText = [self normalizeAccents:bibText];
    
    // Find all citations in the page, and process them.
    // Extract all the fields from the bibtex and translate one-by-one into
    // metadata.  Note that we must accomodate the notoriously, um, 
    // flexible bibtex syntax!  However, to avoid over-matching, we stop
    // once we find a syntax which found any fields.
    FOREACH(NSArray *, r_bibtex, r_bibtexs) {
        OARegex *r_cite = [r_bibtex _0];
        OARegex *r_fld = [r_bibtex _1];
        
        NSArray *citations = [r_cite allMatchingSubstringsOfString:bibText];
        FOREACH(NSString *, citation, citations) {            
            NSMutableDictionary *paperDict = [NSMutableDictionary dictionary];
            NSMutableDictionary *journalDict = [NSMutableDictionary dictionary];
            
            // If we have an abstract, use it
            if (abstract != nil)
                [paperDict setObject:abstract forKey:@"abstract"];
            
            // Load the bib tex fields:
            NSArray *fields = [r_fld allMatchingSubstringsOfString:citation];
            FOREACH(NSString *, field, fields) {
                NSArray *array = [r_fld capturedSubstringsOfString:field];
                NSString *fieldName = [array objectAtIndex:1];
                NSString *fieldValue = [array objectAtIndex:2];
                [self processField:fieldName value:fieldValue paperDict:paperDict journalDict:journalDict];
            } ENDFOREACH;
            
            
            // Special case: set the URL associated with the paper.
            //   If an ACM DOI link is provided, I use that.
            //   Otherwise, I use the URL we loaded originally, modified to avoid
            //   the extra parameters the ACM likes to add.
            NSString *doi = [paperDict objectForKey:@"doi"];
            if ([doi startsWith:@"http://doi.acm.org/"])
                [paperDict setObject:doi forKey:@"url"];
            else {
                NSString *mainUrlText = [mainUrl description];
                NSString *shorterUrl = [r_shorter_url matchedSubstringOfString:mainUrlText];
                [paperDict setObject:(shorterUrl ? shorterUrl : mainUrlText) 
                              forKey:@"url"];
            }
            
            // Special case: add the journal dictionary (if a name was specified)
            DebugLog(@"Journal dictionary=%@", journalDict);
            if ([journalDict objectForKey:@"name"] != nil) {
                DebugLog(@"  name was present.");
                [paperDict setObject:[NSArray arrayWithObject:journalDict] forKey:@"journal"];
            }
            
            // Add in the user specified keywords (if any)
            if ([keywords count] > 0) {
                NSMutableArray *array = [NSMutableArray array];
                FOREACH(NSString *, keyword, keywords) {
                    NSDictionary *dictionary = [NSDictionary dictionaryWithObject:keyword forKey:@"name"];
                    [array addObject:dictionary];
                } ENDFOREACH;
                [paperDict setObject:array forKey:@"keywords"];
            }
            
            // Create the final result.
            NSDictionary *res =
                [NSDictionary dictionaryWithObject:[NSArray arrayWithObject:paperDict] 
                                            forKey:@"papers"];    
            [[self delegate] didImportObject:res];            
        } ENDFOREACH;
    } ENDFOREACH;
}

- (void) cancelImport{
    canceled = YES;
	[self setStatusString: NSLocalizedString(@"Cancelling import of file...", @"Status message shown while cancelling import.")];
}

#pragma mark -
#pragma mark Cleanup methods

// A method to check whether the import finished properly
// and one to get at any errors that resulted. See above for usage of errorCodes.
- (BOOL) successfulCompletion{
	// we simply check whether we caught an error
	return (importError == nil);
}

- (NSError *) importCompletionError{
	return importError;
}

	// let the plugin get rid of any data that needs to be reset for a new import.
- (void) performCleanup{
	[itemsToImport release];
	itemsToImport = nil;
	
	[importedItems release];
	importedItems = nil;
	
	[statusString release];
	statusString = nil;
	
	[importError release];
	importError = nil;
	
}


@end
