a a@sddlZGdddZdS)Nc@s4eZdZddZddZddZddZd d Zd S) CpyTokenizercCs>d|_d|_d|_td|j|j|jf|_|||_dS)Nz[.,;]?$|[.,;]?\sz'[^']*'z"[^"]*"z (%s|%s|%s))Zline_sepZs_quoteZd_quoterecompilereg_ex cpyTokenizetokensselfcpydatar W/Users/arunkse/Documents/Data/MyWork/GitHubRepos/mf-unlock-cobdata/src/cpy_tokenizer.py__init__s zCpyTokenizer.__init__cCs>dd|dD}dd|D}dd|D}d|}|S)NcSs4g|],}t|dkr|ddvr|ddqS))*/H)lenrstrip.0liner r r z)CpyTokenizer.cpyClean.. cSsg|]}|dvr|qS))ZEJECTZSKIP1ZSKIP2ZSKIP3striprr r r rrcSsg|]}t|dkr|qS)r)rrr r r rr )splitjoinrr r r cpyCleans  zCpyTokenizer.cpyCleancCs&||}ddt|j|D}|S)NcSsg|]}|r|qSr r)rtokenr r r rrz,CpyTokenizer.cpyTokenize..)rrrr)r r Z clean_cpydatarr r r rs zCpyTokenizer.cpyTokenizecCs|jr|jd}nd}|SNr)rpopr r r r r getToken!szCpyTokenizer.getTokencCs|r|jd|dSr!)rinsertr#r r r putToken*szCpyTokenizer.putTokenN)__name__ __module__ __qualname__r rrr$r&r r r r rs    r)rrr r r r s