# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # to configure behavior, define $CQL_TEST_HOST to the destination address # and $CQL_TEST_PORT to the associated port. from unittest import TestCase from operator import itemgetter from cqlshlib.cql3handling import CqlRuleSet class TestCqlParsing(TestCase): def test_parse_string_literals(self): for n in ["'eggs'", "'Sausage 1'", "'spam\nspam\n\tsausage'", "''"]: self.assertSequenceEqual(tokens_with_types(CqlRuleSet.lex(n)), [(n, 'quotedStringLiteral')]) self.assertSequenceEqual(tokens_with_types(CqlRuleSet.lex("'eggs'")), [("'eggs'", 'quotedStringLiteral')]) tokens = CqlRuleSet.lex("'spam\nspam\n\tsausage'") tokens = CqlRuleSet.cql_massage_tokens(tokens) self.assertEqual(tokens[0][0], "quotedStringLiteral") tokens = CqlRuleSet.lex("'spam\nspam\n") tokens = CqlRuleSet.cql_massage_tokens(tokens) self.assertEqual(tokens[0][0], "unclosedString") tokens = CqlRuleSet.lex("'foo bar' 'spam\nspam\n") tokens = CqlRuleSet.cql_massage_tokens(tokens) self.assertEqual(tokens[1][0], "unclosedString") def test_parse_pgstring_literals(self): for n in ["$$eggs$$", "$$Sausage 1$$", "$$spam\nspam\n\tsausage$$", "$$$$"]: self.assertSequenceEqual(tokens_with_types(CqlRuleSet.lex(n)), [(n, 'pgStringLiteral')]) self.assertSequenceEqual(tokens_with_types(CqlRuleSet.lex("$$eggs$$")), [("$$eggs$$", 'pgStringLiteral')]) tokens = CqlRuleSet.lex("$$spam\nspam\n\tsausage$$") tokens = CqlRuleSet.cql_massage_tokens(tokens) # [('pgStringLiteral', '$$spam\nspam\n\tsausage$$', (0, 22))] self.assertEqual(tokens[0][0], "pgStringLiteral") tokens = CqlRuleSet.lex("$$spam\nspam\n") tokens = CqlRuleSet.cql_massage_tokens(tokens) # [('unclosedPgString', '$$', (0, 2)), ('identifier', 'spam', (2, 6)), ('identifier', 'spam', (7, 11))] self.assertEqual(tokens[0][0], "unclosedPgString") tokens = CqlRuleSet.lex("$$foo bar$$ $$spam\nspam\n") tokens = CqlRuleSet.cql_massage_tokens(tokens) # [('pgStringLiteral', '$$foo bar$$', (0, 11)), ('unclosedPgString', '$$', (12, 14)), ('identifier', 'spam', (14, 18)), ('identifier', 'spam', (19, 23))] self.assertEqual(tokens[0][0], "pgStringLiteral") self.assertEqual(tokens[1][0], "unclosedPgString") def test_parse_numbers(self): for n in ['6', '398', '18018']: self.assertSequenceEqual(tokens_with_types(CqlRuleSet.lex(n)), [(n, 'wholenumber')]) def test_parse_uuid(self): uuids = ['4feeae80-e9cc-11e4-b571-0800200c9a66', '7142303f-828f-4806-be9e-7a973da0c3f9', 'dff8d435-9ca0-487c-b5d0-b0fe5c5768a8'] for u in uuids: self.assertSequenceEqual(tokens_with_types(CqlRuleSet.lex(u)), [(u, 'uuid')]) def test_comments_in_string_literals(self): comment_strings = ["'sausage -- comment'", "'eggs and spam // comment string'", "'spam eggs sausage and spam /* still in string'"] for s in comment_strings: self.assertSequenceEqual(tokens_with_types(CqlRuleSet.lex(s)), [(s, 'quotedStringLiteral')]) def test_colons_in_string_literals(self): comment_strings = ["'Movie Title: The Movie'", "':a:b:c:'", "'(>>=) :: (Monad m) => m a -> (a -> m b) -> m b'"] for s in comment_strings: self.assertSequenceEqual(tokens_with_types(CqlRuleSet.lex(s)), [(s, 'quotedStringLiteral')]) def test_partial_parsing(self): [parsed] = CqlRuleSet.cql_parse('INSERT INTO ks.test') self.assertSequenceEqual(parsed.matched, []) self.assertSequenceEqual(tokens_with_types(parsed.remainder), [('INSERT', 'reserved_identifier'), ('INTO', 'reserved_identifier'), ('ks', 'identifier'), ('.', 'op'), ('test', 'identifier')]) def test_parse_select(self): parsed = parse_cqlsh_statements('SELECT FROM ks.tab;') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('ks', 'identifier'), ('.', 'op'), ('tab', 'identifier'), (';', 'endtoken')]) parsed = parse_cqlsh_statements('SELECT FROM "MyTable";') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('"MyTable"', 'quotedName'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( 'SELECT FROM tab WHERE foo = 3;') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('tab', 'identifier'), ('WHERE', 'reserved_identifier'), ('foo', 'identifier'), ('=', 'op'), ('3', 'wholenumber'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( 'SELECT FROM tab ORDER BY event_id DESC LIMIT 1000') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('tab', 'identifier'), ('ORDER', 'reserved_identifier'), ('BY', 'reserved_identifier'), ('event_id', 'identifier'), ('DESC', 'reserved_identifier'), ('LIMIT', 'reserved_identifier'), ('1000', 'wholenumber')]) parsed = parse_cqlsh_statements( 'SELECT FROM tab WHERE clustering_column > 200 ' 'AND clustering_column < 400 ALLOW FILTERING') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('tab', 'identifier'), ('WHERE', 'reserved_identifier'), ('clustering_column', 'identifier'), ('>', 'cmp'), ('200', 'wholenumber'), ('AND', 'reserved_identifier'), ('clustering_column', 'identifier'), ('<', 'cmp'), ('400', 'wholenumber'), # 'allow' and 'filtering' are not keywords ('ALLOW', 'reserved_identifier'), ('FILTERING', 'identifier')]) def test_parse_insert(self): parsed = parse_cqlsh_statements('INSERT INTO mytable (x) VALUES (2);') self.assertSequenceEqual(tokens_with_types(parsed), [('INSERT', 'reserved_identifier'), ('INTO', 'reserved_identifier'), ('mytable', 'identifier'), ('(', 'op'), ('x', 'identifier'), (')', 'op'), ('VALUES', 'identifier'), ('(', 'op'), ('2', 'wholenumber'), (')', 'op'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "INSERT INTO mytable (x, y) VALUES (2, 'eggs');") self.assertSequenceEqual(tokens_with_types(parsed), [('INSERT', 'reserved_identifier'), ('INTO', 'reserved_identifier'), ('mytable', 'identifier'), ('(', 'op'), ('x', 'identifier'), (',', 'op'), ('y', 'identifier'), (')', 'op'), ('VALUES', 'identifier'), ('(', 'op'), ('2', 'wholenumber'), (',', 'op'), ("'eggs'", 'quotedStringLiteral'), (')', 'op'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "INSERT INTO mytable (x, y) VALUES (2, 'eggs');") self.assertSequenceEqual(tokens_with_types(parsed), [('INSERT', 'reserved_identifier'), ('INTO', 'reserved_identifier'), ('mytable', 'identifier'), ('(', 'op'), ('x', 'identifier'), (',', 'op'), ('y', 'identifier'), (')', 'op'), ('VALUES', 'identifier'), ('(', 'op'), ('2', 'wholenumber'), (',', 'op'), ("'eggs'", 'quotedStringLiteral'), (')', 'op'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "INSERT INTO mytable (ids) VALUES " "(7ee251da-af52-49a4-97f4-3f07e406c7a7) " "USING TTL 86400;") self.assertSequenceEqual(tokens_with_types(parsed), [('INSERT', 'reserved_identifier'), ('INTO', 'reserved_identifier'), ('mytable', 'identifier'), ('(', 'op'), ('ids', 'identifier'), (')', 'op'), ('VALUES', 'identifier'), ('(', 'op'), ('7ee251da-af52-49a4-97f4-3f07e406c7a7', 'uuid'), (')', 'op'), ('USING', 'reserved_identifier'), ('TTL', 'identifier'), ('86400', 'wholenumber'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "INSERT INTO test_table (username) VALUES ('Albert') " "USING TIMESTAMP 1240003134 AND TTL 600;") self.assertSequenceEqual(tokens_with_types(parsed), [('INSERT', 'reserved_identifier'), ('INTO', 'reserved_identifier'), ('test_table', 'identifier'), ('(', 'op'), ('username', 'identifier'), (')', 'op'), ('VALUES', 'identifier'), ('(', 'op'), ("'Albert'", 'quotedStringLiteral'), (')', 'op'), ('USING', 'reserved_identifier'), ('TIMESTAMP', 'identifier'), ('1240003134', 'wholenumber'), ('AND', 'reserved_identifier'), ('TTL', 'identifier'), ('600', 'wholenumber'), (';', 'endtoken')]) def test_parse_update(self): parsed = parse_cqlsh_statements( "UPDATE tab SET x = 15 WHERE y = 'eggs';") self.assertSequenceEqual(tokens_with_types(parsed), [('UPDATE', 'reserved_identifier'), ('tab', 'identifier'), ('SET', 'reserved_identifier'), ('x', 'identifier'), ('=', 'op'), ('15', 'wholenumber'), ('WHERE', 'reserved_identifier'), ('y', 'identifier'), ('=', 'op'), ("'eggs'", 'quotedStringLiteral'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "UPDATE tab USING TTL 432000 SET x = 15 WHERE y = 'eggs';") self.assertSequenceEqual(tokens_with_types(parsed), [('UPDATE', 'reserved_identifier'), ('tab', 'identifier'), ('USING', 'reserved_identifier'), ('TTL', 'identifier'), ('432000', 'wholenumber'), ('SET', 'reserved_identifier'), ('x', 'identifier'), ('=', 'op'), ('15', 'wholenumber'), ('WHERE', 'reserved_identifier'), ('y', 'identifier'), ('=', 'op'), ("'eggs'", 'quotedStringLiteral'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "UPDATE tab SET x = 15, y = 'sausage' " "WHERE y = 'eggs';") self.assertSequenceEqual(tokens_with_types(parsed), [('UPDATE', 'reserved_identifier'), ('tab', 'identifier'), ('SET', 'reserved_identifier'), ('x', 'identifier'), ('=', 'op'), ('15', 'wholenumber'), (',', 'op'), ('y', 'identifier'), ('=', 'op'), ("'sausage'", 'quotedStringLiteral'), ('WHERE', 'reserved_identifier'), ('y', 'identifier'), ('=', 'op'), ("'eggs'", 'quotedStringLiteral'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "UPDATE tab SET x = 15 " "WHERE y IN ('eggs', 'sausage', 'spam');") self.assertSequenceEqual(tokens_with_types(parsed), [('UPDATE', 'reserved_identifier'), ('tab', 'identifier'), ('SET', 'reserved_identifier'), ('x', 'identifier'), ('=', 'op'), ('15', 'wholenumber'), ('WHERE', 'reserved_identifier'), ('y', 'identifier'), ('IN', 'reserved_identifier'), ('(', 'op'), ("'eggs'", 'quotedStringLiteral'), (',', 'op'), ("'sausage'", 'quotedStringLiteral'), (',', 'op'), ("'spam'", 'quotedStringLiteral'), (')', 'op'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "UPDATE tab SET x = 15 " "WHERE y = 'spam' IF z = 'sausage';") self.assertSequenceEqual(tokens_with_types(parsed), [('UPDATE', 'reserved_identifier'), ('tab', 'identifier'), ('SET', 'reserved_identifier'), ('x', 'identifier'), ('=', 'op'), ('15', 'wholenumber'), ('WHERE', 'reserved_identifier'), ('y', 'identifier'), ('=', 'op'), ("'spam'", 'quotedStringLiteral'), ('IF', 'reserved_identifier'), ('z', 'identifier'), ('=', 'op'), ("'sausage'", 'quotedStringLiteral'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "UPDATE tab SET x = 15 WHERE y = 'spam' " "IF z = 'sausage' AND w = 'spam';") self.assertSequenceEqual(tokens_with_types(parsed), [('UPDATE', 'reserved_identifier'), ('tab', 'identifier'), ('SET', 'reserved_identifier'), ('x', 'identifier'), ('=', 'op'), ('15', 'wholenumber'), ('WHERE', 'reserved_identifier'), ('y', 'identifier'), ('=', 'op'), ("'spam'", 'quotedStringLiteral'), ('IF', 'reserved_identifier'), ('z', 'identifier'), ('=', 'op'), ("'sausage'", 'quotedStringLiteral'), ('AND', 'reserved_identifier'), ('w', 'identifier'), ('=', 'op'), ("'spam'", 'quotedStringLiteral'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "UPDATE tab SET x = 15 WHERE y = 'spam' IF EXISTS") self.assertSequenceEqual(tokens_with_types(parsed), [('UPDATE', 'reserved_identifier'), ('tab', 'identifier'), ('SET', 'reserved_identifier'), ('x', 'identifier'), ('=', 'op'), ('15', 'wholenumber'), ('WHERE', 'reserved_identifier'), ('y', 'identifier'), ('=', 'op'), ("'spam'", 'quotedStringLiteral'), ('IF', 'reserved_identifier'), ('EXISTS', 'identifier')]) def test_parse_delete(self): parsed = parse_cqlsh_statements( "DELETE FROM songs WHERE songid = 444;") self.assertSequenceEqual(tokens_with_types(parsed), [('DELETE', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('songs', 'identifier'), ('WHERE', 'reserved_identifier'), ('songid', 'identifier'), ('=', 'op'), ('444', 'wholenumber'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "DELETE FROM songs WHERE name IN " "('Yellow Submarine', 'Eleanor Rigby');") self.assertSequenceEqual(tokens_with_types(parsed), [('DELETE', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('songs', 'identifier'), ('WHERE', 'reserved_identifier'), ('name', 'identifier'), ('IN', 'reserved_identifier'), ('(', 'op'), ("'Yellow Submarine'", 'quotedStringLiteral'), (',', 'op'), ("'Eleanor Rigby'", 'quotedStringLiteral'), (')', 'op'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "DELETE task_map ['2014-12-25'] FROM tasks WHERE user_id = 'Santa';") self.assertSequenceEqual(tokens_with_types(parsed), [('DELETE', 'reserved_identifier'), ('task_map', 'identifier'), ('[', 'brackets'), ("'2014-12-25'", 'quotedStringLiteral'), (']', 'brackets'), ('FROM', 'reserved_identifier'), ('tasks', 'identifier'), ('WHERE', 'reserved_identifier'), ('user_id', 'identifier'), ('=', 'op'), ("'Santa'", 'quotedStringLiteral'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "DELETE my_list[0] FROM lists WHERE user_id = 'Jim';") self.assertSequenceEqual(tokens_with_types(parsed), [('DELETE', 'reserved_identifier'), ('my_list', 'identifier'), ('[', 'brackets'), ('0', 'wholenumber'), (']', 'brackets'), ('FROM', 'reserved_identifier'), ('lists', 'identifier'), ('WHERE', 'reserved_identifier'), ('user_id', 'identifier'), ('=', 'op'), ("'Jim'", 'quotedStringLiteral'), (';', 'endtoken')]) def test_parse_batch(self): pass def test_parse_create_keyspace(self): parsed = parse_cqlsh_statements( "CREATE KEYSPACE ks WITH REPLICATION = " "{'class': 'SimpleStrategy', 'replication_factor': 1};") self.assertSequenceEqual(tokens_with_types(parsed), [('CREATE', 'reserved_identifier'), ('KEYSPACE', 'reserved_identifier'), ('ks', 'identifier'), ('WITH', 'reserved_identifier'), ('REPLICATION', 'identifier'), ('=', 'op'), ('{', 'brackets'), ("'class'", 'quotedStringLiteral'), (':', 'colon'), ("'SimpleStrategy'", 'quotedStringLiteral'), (',', 'op'), ("'replication_factor'", 'quotedStringLiteral'), (':', 'colon'), ('1', 'wholenumber'), ('}', 'brackets'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( 'CREATE KEYSPACE "Cql_test_KS" WITH REPLICATION = ' "{'class': 'NetworkTopologyStrategy', 'dc1' : 3, 'dc2': 2};") self.assertSequenceEqual(tokens_with_types(parsed), [('CREATE', 'reserved_identifier'), ('KEYSPACE', 'reserved_identifier'), ('"Cql_test_KS"', 'quotedName'), ('WITH', 'reserved_identifier'), ('REPLICATION', 'identifier'), ('=', 'op'), ('{', 'brackets'), ("'class'", 'quotedStringLiteral'), (':', 'colon'), ("'NetworkTopologyStrategy'", 'quotedStringLiteral'), (',', 'op'), ("'dc1'", 'quotedStringLiteral'), (':', 'colon'), ('3', 'wholenumber'), (',', 'op'), ("'dc2'", 'quotedStringLiteral'), (':', 'colon'), ('2', 'wholenumber'), ('}', 'brackets'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "CREATE KEYSPACE ks WITH REPLICATION = " "{'class': 'NetworkTopologyStrategy', 'dc1': 3} AND " "DURABLE_WRITES = false;") self.assertSequenceEqual(tokens_with_types(parsed), [('CREATE', 'reserved_identifier'), ('KEYSPACE', 'reserved_identifier'), ('ks', 'identifier'), ('WITH', 'reserved_identifier'), ('REPLICATION', 'identifier'), ('=', 'op'), ('{', 'brackets'), ("'class'", 'quotedStringLiteral'), (':', 'colon'), ("'NetworkTopologyStrategy'", 'quotedStringLiteral'), (',', 'op'), ("'dc1'", 'quotedStringLiteral'), (':', 'colon'), ('3', 'wholenumber'), ('}', 'brackets'), ('AND', 'reserved_identifier'), # 'DURABLE_WRITES' is not a keyword ('DURABLE_WRITES', 'identifier'), ('=', 'op'), ('false', 'identifier'), (';', 'endtoken')]) def test_parse_drop_keyspace(self): parsed = parse_cqlsh_statements( 'DROP KEYSPACE ks;') self.assertSequenceEqual(tokens_with_types(parsed), [('DROP', 'reserved_identifier'), ('KEYSPACE', 'reserved_identifier'), ('ks', 'identifier'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( 'DROP SCHEMA ks;') self.assertSequenceEqual(tokens_with_types(parsed), [('DROP', 'reserved_identifier'), ('SCHEMA', 'reserved_identifier'), ('ks', 'identifier'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( 'DROP KEYSPACE IF EXISTS "My_ks";') self.assertSequenceEqual(tokens_with_types(parsed), [('DROP', 'reserved_identifier'), ('KEYSPACE', 'reserved_identifier'), ('IF', 'reserved_identifier'), ('EXISTS', 'identifier'), ('"My_ks"', 'quotedName'), (';', 'endtoken')]) def test_parse_create_table(self): pass def test_parse_drop_table(self): pass def test_parse_truncate(self): pass def test_parse_alter_table(self): pass def test_parse_use(self): pass def test_parse_create_index(self): parsed = parse_cqlsh_statements( 'CREATE INDEX idx ON ks.tab (i);') self.assertSequenceEqual(tokens_with_types(parsed), (('CREATE', 'reserved_identifier'), ('INDEX', 'reserved_identifier'), ('idx', 'identifier'), ('ON', 'reserved_identifier'), ('ks', 'identifier'), ('.', 'op'), ('tab', 'identifier'), ('(', 'op'), ('i', 'identifier'), (')', 'op'), (';', 'endtoken'))) parsed = parse_cqlsh_statements( 'CREATE INDEX idx ON ks.tab (i) IF NOT EXISTS;') self.assertSequenceEqual(tokens_with_types(parsed), (('CREATE', 'reserved_identifier'), ('INDEX', 'reserved_identifier'), ('idx', 'identifier'), ('ON', 'reserved_identifier'), ('ks', 'identifier'), ('.', 'op'), ('tab', 'identifier'), ('(', 'op'), ('i', 'identifier'), (')', 'op'), ('IF', 'reserved_identifier'), ('NOT', 'reserved_identifier'), ('EXISTS', 'identifier'), (';', 'endtoken'))) parsed = parse_cqlsh_statements( 'CREATE INDEX idx ON tab (KEYS(i));') self.assertSequenceEqual(tokens_with_types(parsed), (('CREATE', 'reserved_identifier'), ('INDEX', 'reserved_identifier'), ('idx', 'identifier'), ('ON', 'reserved_identifier'), ('tab', 'identifier'), ('(', 'op'), ('KEYS', 'identifier'), ('(', 'op'), ('i', 'identifier'), (')', 'op'), (')', 'op'), (';', 'endtoken'))) parsed = parse_cqlsh_statements( 'CREATE INDEX idx ON ks.tab FULL(i);') self.assertSequenceEqual(tokens_with_types(parsed), [('CREATE', 'reserved_identifier'), ('INDEX', 'reserved_identifier'), ('idx', 'identifier'), ('ON', 'reserved_identifier'), ('ks', 'identifier'), ('.', 'op'), ('tab', 'identifier'), ('FULL', 'reserved_identifier'), ('(', 'op'), ('i', 'identifier'), (')', 'op'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( 'CREATE CUSTOM INDEX idx ON ks.tab (i);') self.assertSequenceEqual(tokens_with_types(parsed), [('CREATE', 'reserved_identifier'), ('CUSTOM', 'identifier'), ('INDEX', 'reserved_identifier'), ('idx', 'identifier'), ('ON', 'reserved_identifier'), ('ks', 'identifier'), ('.', 'op'), ('tab', 'identifier'), ('(', 'op'), ('i', 'identifier'), (')', 'op'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "CREATE INDEX idx ON ks.tab (i) USING " "'org.custom.index.MyIndexClass';") self.assertSequenceEqual(tokens_with_types(parsed), [('CREATE', 'reserved_identifier'), ('INDEX', 'reserved_identifier'), ('idx', 'identifier'), ('ON', 'reserved_identifier'), ('ks', 'identifier'), ('.', 'op'), ('tab', 'identifier'), ('(', 'op'), ('i', 'identifier'), (')', 'op'), ('USING', 'reserved_identifier'), ("'org.custom.index.MyIndexClass'", 'quotedStringLiteral'), (';', 'endtoken')]) parsed = parse_cqlsh_statements( "CREATE INDEX idx ON ks.tab (i) WITH OPTIONS = " "{'storage': '/mnt/ssd/indexes/'};") self.assertSequenceEqual(tokens_with_types(parsed), [('CREATE', 'reserved_identifier'), ('INDEX', 'reserved_identifier'), ('idx', 'identifier'), ('ON', 'reserved_identifier'), ('ks', 'identifier'), ('.', 'op'), ('tab', 'identifier'), ('(', 'op'), ('i', 'identifier'), (')', 'op'), ('WITH', 'reserved_identifier'), ('OPTIONS', 'identifier'), ('=', 'op'), ('{', 'brackets'), ("'storage'", 'quotedStringLiteral'), (':', 'colon'), ("'/mnt/ssd/indexes/'", 'quotedStringLiteral'), ('}', 'brackets'), (';', 'endtoken')]) def test_parse_drop_index(self): pass def test_parse_select_token(self): pass def test_strip_comment_blocks_from_input(self): parsed = parse_cqlsh_statements('SELECT FROM /* comment block */ "MyTable";') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('"MyTable"', 'quotedName'), (';', 'endtoken')]) parsed = parse_cqlsh_statements('SELECT FROM /* \n comment block starts here; \n and continues here \n */ "MyTable";') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('"MyTable"', 'quotedName'), (';', 'endtoken')]) parsed = parse_cqlsh_statements(''' SELECT FROM /* comment block starts here; and continues here */ "MyTable"; ''') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('"MyTable"', 'quotedName'), (';', 'endtoken')]) parsed = parse_cqlsh_statements(''' /* comment block */ SELECT FROM "MyTable"; ''') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('"MyTable"', 'quotedName'), (';', 'endtoken')]) parsed = parse_cqlsh_statements(''' /* comment block */ /* another comment */ SELECT FROM /* comment block starts here; and continues here */ "MyTable"; ''') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('"MyTable"', 'quotedName'), (';', 'endtoken')]) parsed = parse_cqlsh_statements(''' SELECT FROM "/*MyTable*/"; ''') self.assertSequenceEqual(tokens_with_types(parsed), [('SELECT', 'reserved_identifier'), ('FROM', 'reserved_identifier'), ('"/*MyTable*/"', 'quotedName'), (';', 'endtoken')]) parse_cqlsh_statements(''' */ SELECT FROM "MyTable"; ''') self.assertRaises(SyntaxError) def parse_cqlsh_statements(text): """ Runs its argument through the sequence of parsing steps that cqlsh takes its input through. Currently does not handle batch statements. """ # based on onecmd statements, _ = CqlRuleSet.cql_split_statements(text) # stops here. For regular cql commands, onecmd just splits it and sends it # off to the cql engine; parsing only happens for cqlsh-specific stmts. return strip_final_empty_items(statements)[0] def tokens_with_types(lexed): for x in lexed: assert len(x) > 2, lexed return tuple(itemgetter(1, 0)(token) for token in lexed) def strip_final_empty_items(xs): """ Returns its a copy of argument as a list, but with any terminating subsequence of falsey values removed. >>> strip_final_empty_items([[3, 4], [5, 6, 7], [], [], [1], []]) [[3, 4], [5, 6, 7], [], [], [1]] """ rv = list(xs) while rv and not rv[-1]: rv = rv[:-1] return rv