o a@sddlZGdddZdS)Nc@s4eZdZddZddZddZddZd d Zd S) CpyTokenizercCs>d|_d|_d|_td|j|j|jf|_|||_dS)Nz[.,;]?$|[.,;]?\sz'[^']*'z"[^"]*"z (%s|%s|%s))Zline_sepZs_quoteZd_quoterecompilereg_ex cpyTokenizetokensselfcpydatar ?d:\MyWork\GitHubRepos\mf-unlock-cobdata-v0\src\cpy_tokenizer.py__init__s zCpyTokenizer.__init__cCs>dd|dD}dd|D}dd|D}d|}|S)NcSs4g|]}t|dkr|ddvr|ddqS))*/H)lenrstrip.0liner r r s4z)CpyTokenizer.cpyClean.. cSsg|] }|dvr|qS))ZEJECTZSKIP1ZSKIP2ZSKIP3striprr r r rcSsg|] }t|dkr|qS)r)rrr r r rr )splitjoinrr r r cpyCleans  zCpyTokenizer.cpyCleancCs&||}ddt|j|D}|S)NcSsg|] }|r|qSr r)rtokenr r r rrz,CpyTokenizer.cpyTokenize..)rrrr)r r Z clean_cpydatarr r r rs zCpyTokenizer.cpyTokenizecCs|jr |jd}|Sd}|SNr)rpopr r r r r getToken!s  zCpyTokenizer.getTokencCs|r |jd|dSdSr!)rinsertr#r r r putToken*szCpyTokenizer.putTokenN)__name__ __module__ __qualname__r rrr$r&r r r r rs    r)rrr r r r s