in your home directory (instead of stdout)."
+ " This will overwrite any existing file with the same name";
// Messages string constants
public static final String DUPLICATE_COLUMN_KEY_DETECTED_FOR_TABLE_SCHEMA =
"Duplicate column key '%s' detected for table schema '%s'. Original column '%s'."
+ " Duplicate column '%s'.";
private static final String NEW_SCHEMA_VERSION_GENERATED_MESSAGE =
"New schema '%s', version '%s' generated.";
private static final String REMOVED_SCHEMA_MESSAGE = "Removed schema '%s'.";
private static MongoClient client;
static {
ARCHIVE_VERSION = getArchiveVersion();
LIBRARY_NAME = getLibraryName();
HELP_OPTION = buildHelpOption();
VERSION_OPTION = buildVersionOption();
COMMAND_OPTIONS = buildCommandOptions();
REQUIRED_OPTIONS = buildRequiredOptions();
OPTIONAL_OPTIONS = buildOptionalOptions();
// Add all option types.
COMPLETE_OPTIONS = new Options();
COMPLETE_OPTIONS.addOptionGroup(COMMAND_OPTIONS);
REQUIRED_OPTIONS.forEach(COMPLETE_OPTIONS::addOption);
OPTIONAL_OPTIONS.forEach(COMPLETE_OPTIONS::addOption);
// Add options to check for 'help' or 'version'.
HELP_VERSION_OPTIONS = new Options()
.addOption(HELP_OPTION)
.addOption(VERSION_OPTION);
}
/**
* Performs schema commands via the command line.
*
* -a,--tls-allow-invalid-hostnames The indicator of whether to allow invalid
* hostnames when connecting to DocumentDB.
* Default: false.
* -b,--list-tables Lists the SQL table names in a schema.
* -d,--database <database-name> The name of the database for the schema
* operations. Required.
* -e,--export <[table-name[,...]]> Exports the schema to for SQL tables named
* [<table-name>[,<table-name>[…]]]. If no
* <table-name> are given, all table schema will
* be exported. By default, the schema is
* written to stdout. Use the --output option to
* write to a file. The output format is JSON.
* -g,--generate-new Generates a new schema for the database. This
* will have the effect of replacing an existing
* schema of the same name, if it exists.
* -h,--help Prints the command line syntax.
* -i,--import <file-name> Imports the schema from <file-name> in your
* home directory. The schema will be imported
* using the <schema-name> and a new version
* will be added - replacing the existing
* schema. The expected input format is JSON.
* -l,--list-schema Lists the schema names, version and table
* names available in the schema repository.
* -m,--scan-method <method> The scan method to sample documents from the
* collections. One of: random, idForward,
* idReverse, or all. Used in conjunction with
* the --generate-new command. Default: random.
* -n,--schema-name <schema-name> The name of the schema. Default: _default.
* -o,--output <file-name> Write the exported schema to <file-name> in
* your home directory (instead of stdout). This
* will overwrite any existing file with the
* same name
* -p,--password <password> The password for the user performing the
* schema operations. Optional. If this option
* is not provided, the end-user will be
* prompted to enter the password directly.
* -r,--remove Removes the schema from storage for schema
* given by -m <schema-name>, or for schema
* '_default', if not provided.
* -s,--server <host-name> The hostname and optional port number
* (default: 27017) in the format
* hostname[:port]. Required.
* -t,--tls The indicator of whether to use TLS
* encryption when connecting to DocumentDB.
* Default: false.
* -u,--user <user-name> The name of the user performing the schema
* operations. Required. Note: the user will
* require readWrite role on the <database-name>
* where the schema are stored if creating or
* modifying schema.
* --version Prints the version number of the command.
* -x,--scan-limit <max-documents> The maximum number of documents to sample in
* each collection. Used in conjunction with the
* --generate-new command. Default: 1000.,
*
* @param args the command line arguments.
*/
public static void main(final String[] args) {
try {
final StringBuilder output = new StringBuilder();
handleCommandLine(args, output);
if (output.length() > 0) {
LOGGER.error("{}", output);
}
} catch (SQLException e) {
LOGGER.error(e.getMessage(), e);
} catch (Exception e) {
LOGGER.error(
"Unexpected exception: '{}'",
e.getMessage(),
e);
}
}
static void handleCommandLine(final String[] args, final StringBuilder output)
throws SQLException {
if (handledHelpOrVersionOption(args, output)) {
return;
}
try {
final CommandLineParser parser = new DefaultParser();
final CommandLine commandLine = parser.parse(COMPLETE_OPTIONS, args);
final DocumentDbConnectionProperties properties = new DocumentDbConnectionProperties();
if (!tryGetConnectionProperties(commandLine, properties, output)) {
return;
}
performCommand(commandLine, properties, output);
} catch (MissingOptionException e) {
output.append(e.getMessage()).append(String.format("%n"));
printHelp(output);
} catch (ParseException e) {
output.append(e.getMessage());
} catch (Exception e) {
output.append(e.getClass().getSimpleName())
.append(": ")
.append(e.getMessage());
} finally {
closeClient();
}
}
private static void performCommand(
final CommandLine commandLine,
final DocumentDbConnectionProperties properties,
final StringBuilder output)
throws SQLException {
switch (COMMAND_OPTIONS.getSelected()) {
case GENERATE_NAME_OPTION_FLAG: // --generate-new
performGenerateNew(properties, output);
break;
case REMOVE_OPTION_FLAG: // --remove
performRemove(properties, output);
break;
case LIST_OPTION_FLAG: // --list-schema
performListSchema(properties, output);
break;
case LIST_TABLES_OPTION_FLAG: // --list-tables
performListTables(properties, output);
break;
case EXPORT_OPTION_FLAG: // --export
performExport(commandLine, properties, output);
break;
case IMPORT_OPTION_FLAG: // --import
performImport(commandLine, properties, output);
break;
default:
output.append(SqlError.lookup(SqlError.UNSUPPORTED_PROPERTY,
COMMAND_OPTIONS.getSelected()));
break;
}
}
private static MongoClient getMongoClient(final DocumentDbConnectionProperties properties) {
if (client == null) {
client = properties.createMongoClient();
}
return client;
}
private static void closeClient() {
if (client != null) {
client.close();
client = null;
}
}
private static void performImport(
final CommandLine commandLine,
final DocumentDbConnectionProperties properties,
final StringBuilder output) throws DuplicateKeyException {
final File importFile = tryGetImportFile(commandLine, output);
if (importFile == null) {
return;
}
final List tableSchemaList = tryReadTableSchemaList(importFile, output);
if (tableSchemaList == null) {
return;
}
final List schemaTableList = tryGetSchemaTableList(
tableSchemaList, output);
if (schemaTableList == null) {
return;
}
updateTableSchema(properties, schemaTableList, output);
}
private static void updateTableSchema(
final DocumentDbConnectionProperties properties,
final List schemaTableList,
final StringBuilder output) {
try {
DocumentDbDatabaseSchemaMetadata.update(
properties,
properties.getSchemaName(),
schemaTableList,
getMongoClient(properties));
} catch (SQLException | DocumentDbSchemaSecurityException e) {
output.append(e.getClass().getSimpleName())
.append(" ")
.append(e.getMessage());
}
}
private static List tryReadTableSchemaList(
final File importFile,
final StringBuilder output) {
final List tableSchemaList;
try {
tableSchemaList = JSON_OBJECT_MAPPER.readValue(importFile,
new TypeReference>() { });
} catch (IOException e) {
output.append(e.getClass().getSimpleName())
.append(" ")
.append(e.getMessage());
return null;
}
return tableSchemaList;
}
private static List tryGetSchemaTableList(
final List tableSchemaList, final StringBuilder output) {
final List schemaTableList;
try {
schemaTableList = tableSchemaList.stream()
.map(tableSchema -> new DocumentDbSchemaTable(
tableSchema.getSqlName(),
tableSchema.getCollectionName(),
tableSchema.getColumns().stream()
.collect(Collectors.toMap(
DocumentDbSchemaColumn::getSqlName,
c -> c,
(c1, c2) -> throwingDuplicateMergeOnColumn(c1, c2,
tableSchema.getSqlName()),
LinkedHashMap::new))))
.collect(Collectors.toList());
} catch (IllegalStateException e) {
output.append(e.getMessage());
return null;
}
return schemaTableList;
}
private static DocumentDbSchemaColumn throwingDuplicateMergeOnColumn(
final DocumentDbSchemaColumn c1,
final DocumentDbSchemaColumn c2,
final String sqlName) {
throw new IllegalStateException(String.format(DUPLICATE_COLUMN_KEY_DETECTED_FOR_TABLE_SCHEMA,
c1.getSqlName(),
sqlName,
c1,
c2));
}
private static File tryGetImportFile(
final CommandLine commandLine,
final StringBuilder output) {
final String importFileName = commandLine.getOptionValue(IMPORT_OPTION_FLAG, null);
if (isNullOrEmpty(importFileName)) {
output.append(String.format("Option '-%s' requires a file name argument.", IMPORT_OPTION_FLAG));
return null;
}
final Path importFilePath = USER_HOME_PATH.resolve(importFileName);
if (!importFilePath.toFile().exists()) {
output.append(String.format("Import file '%s' not found in your user's home folder.", importFileName));
return null;
}
return importFilePath.toFile();
}
private static void performExport(
final CommandLine commandLine,
final DocumentDbConnectionProperties properties,
final StringBuilder output) throws SQLException {
// Determine if output file is required.
final File outputFile;
if (commandLine.hasOption(OUTPUT_OPTION_FLAG)) {
outputFile = tryGetOutputFile(commandLine, output);
if (outputFile == null) {
return;
}
} else {
outputFile = null;
}
final String[] requestedTableNames = commandLine.getOptionValues(EXPORT_OPTION_FLAG);
final List requestedTableList = requestedTableNames != null
? Arrays.asList(requestedTableNames)
: new ArrayList<>();
final DocumentDbDatabaseSchemaMetadata schema = DocumentDbDatabaseSchemaMetadata.get(
properties,
properties.getSchemaName(),
VERSION_LATEST_OR_NONE,
getMongoClient(properties));
if (schema == null) {
// No schema to export.
return;
}
final Set availTableSet = schema.getTableSchemaMap().keySet();
if (requestedTableList.isEmpty()) {
requestedTableList.addAll(availTableSet);
} else if (verifyRequestedTablesExist(requestedTableList, availTableSet, output)) {
return;
}
final List tableSchemaList = requestedTableList.stream()
.map(tableName -> new TableSchema(schema.getTableSchemaMap().get(tableName)))
.sorted(Comparator.comparing(TableSchema::getSqlName))
.collect(Collectors.toList());
try {
writeTableSchemas(tableSchemaList, outputFile, output);
} catch (IOException e) {
output.append(e.getClass().getSimpleName())
.append(" ")
.append(e.getMessage());
}
}
private static boolean verifyRequestedTablesExist(
final List requestedTableList,
final Set availTableNames,
final StringBuilder output) {
if (!availTableNames.containsAll(requestedTableList)) {
final List unknownTables = requestedTableList.stream()
.filter(name -> !availTableNames.contains(name))
.collect(Collectors.toList());
output.append("Requested table name(s) are not recognized in schema: ")
.append(Strings.join(unknownTables, ','))
.append(String.format("%n"))
.append("Available table names: ")
.append(Strings.join(availTableNames, ','));
return true;
}
return false;
}
private static void writeTableSchemas(
final List tables, final File outputFile, final StringBuilder output) throws IOException {
try (Writer writer = outputFile != null
? new OutputStreamWriter(Files.newOutputStream(outputFile.toPath()), StandardCharsets.UTF_8)
: new StringBuilderWriter(output)) {
JSON_OBJECT_MAPPER.writeValue(writer, tables);
}
}
private static File tryGetOutputFile(final CommandLine commandLine, final StringBuilder output) {
if (!USER_HOME_PATH.toFile().exists()) {
output.append("User's home directory does not exist.");
return null;
}
final String outputFileName = commandLine.getOptionValue(OUTPUT_OPTION_FLAG, null);
if (isNullOrEmpty(outputFileName)) {
output.append("Output file name argument must not be empty.");
return null;
}
final Path fileNamePath = Paths.get(outputFileName).getFileName();
final File outputFile = USER_HOME_PATH.resolve(fileNamePath).toAbsolutePath().toFile();
if (outputFile.isDirectory()) {
output.append("Output file name must not be a directory.");
return null;
}
return outputFile;
}
private static void performListSchema(
final DocumentDbConnectionProperties properties,
final StringBuilder output) throws SQLException {
final List schemas = DocumentDbDatabaseSchemaMetadata.getSchemaList(
properties, getMongoClient(properties));
for (DocumentDbSchema schema : schemas) {
output.append(String.format("Name=%s, Version=%d, SQL Name=%s, Modified=%s%n",
maybeQuote(schema.getSchemaName()),
schema.getSchemaVersion(),
maybeQuote(schema.getSqlName()),
new SimpleDateFormat(DATE_FORMAT_PATTERN)
.format(schema.getModifyDate()))
);
}
}
private static void performListTables(
final DocumentDbConnectionProperties properties,
final StringBuilder output) throws SQLException {
final DocumentDbDatabaseSchemaMetadata schema = DocumentDbDatabaseSchemaMetadata.get(
properties,
properties.getSchemaName(),
VERSION_LATEST_OR_NONE,
getMongoClient(properties));
if (schema != null) {
final List sortedTableNames = schema.getTableSchemaMap().keySet().stream()
.sorted()
.collect(Collectors.toList());
for (String tableName : sortedTableNames) {
output.append(String.format("%s%n", tableName));
}
}
}
@VisibleForTesting
static String maybeQuote(final String value) {
return StringEscapeUtils.escapeCsv(value);
}
private static void performRemove(
final DocumentDbConnectionProperties properties,
final StringBuilder output) throws SQLException {
DocumentDbDatabaseSchemaMetadata.remove(
properties,
properties.getSchemaName(),
getMongoClient(properties));
output.append(String.format(REMOVED_SCHEMA_MESSAGE, properties.getSchemaName()));
}
private static void performGenerateNew(
final DocumentDbConnectionProperties properties,
final StringBuilder output) throws SQLException {
final DocumentDbDatabaseSchemaMetadata schema = DocumentDbDatabaseSchemaMetadata.get(
properties,
properties.getSchemaName(),
VERSION_NEW,
getMongoClient(properties));
if (schema != null) {
output.append(String.format(NEW_SCHEMA_VERSION_GENERATED_MESSAGE,
schema.getSchemaName(),
schema.getSchemaVersion()));
}
}
@VisibleForTesting
static boolean tryGetConnectionProperties(
final CommandLine commandLine,
final DocumentDbConnectionProperties properties,
final StringBuilder output) {
properties.setHostname(commandLine.getOptionValue(SERVER_OPTION_FLAG));
properties.setDatabase(commandLine.getOptionValue(DATABASE_OPTION_FLAG));
properties.setUser(commandLine.getOptionValue(USER_OPTION_FLAG));
if (!trySetPassword(commandLine, properties, output)) {
return false;
}
properties.setTlsEnabled(String.valueOf(commandLine.hasOption(TLS_OPTION_FLAG)));
properties.setTlsAllowInvalidHostnames(String.valueOf(commandLine.hasOption(TLS_ALLOW_INVALID_HOSTNAMES_OPTION_FLAG)));
properties.setMetadataScanMethod(commandLine.getOptionValue(
SCAN_METHOD_OPTION_FLAG,
DocumentDbConnectionProperty.METADATA_SCAN_METHOD.getDefaultValue()));
properties.setMetadataScanLimit(commandLine.getOptionValue(
SCAN_LIMIT_OPTION_FLAG,
DocumentDbConnectionProperty.METADATA_SCAN_LIMIT.getDefaultValue()));
properties.setSchemaName(commandLine.getOptionValue(
SCHEMA_NAME_OPTION_FLAG,
DocumentDbConnectionProperty.SCHEMA_NAME.getDefaultValue()));
return true;
}
private static boolean trySetPassword(final CommandLine commandLine,
final DocumentDbConnectionProperties properties, final StringBuilder output) {
if (commandLine.hasOption(PASSWORD_OPTION_FLAG)) {
properties.setPassword(commandLine.getOptionValue(PASSWORD_OPTION_FLAG));
} else {
return trySetPasswordFromPromptInput(properties, output);
}
return true;
}
private static boolean trySetPasswordFromPromptInput(
final DocumentDbConnectionProperties properties,
final StringBuilder output) {
final String passwordPrompt = SqlError.lookup(SqlError.PASSWORD_PROMPT);
final Console console = System.console();
char[] password = null;
if (console != null) {
password = console.readPassword(passwordPrompt);
} else {
output.append("No console available.");
}
if (password == null || password.length == 0) {
output.append(SqlError.lookup(SqlError.MISSING_PASSWORD));
return false;
}
properties.setPassword(new String(password));
return true;
}
private static boolean handledHelpOrVersionOption(
final String[] args,
final StringBuilder output) throws SQLException {
final CommandLineParser parser = new DefaultParser();
final CommandLine commandLine;
try {
commandLine = parser.parse(HELP_VERSION_OPTIONS, args, true);
} catch (ParseException e) {
throw new SQLException(e.getMessage(), e);
}
if (commandLine.hasOption(HELP_OPTION_NAME)) {
printHelp(output);
return true;
} else if (commandLine.hasOption(VERSION_OPTION_NAME)) {
output.append(String.format("%s: version %s", LIBRARY_NAME, ARCHIVE_VERSION));
return true;
}
return false;
}
private static void printHelp(final StringBuilder output) {
final StringWriter stringWriter = new StringWriter();
final PrintWriter printWriter = new PrintWriter(stringWriter);
final HelpFormatter formatter = new HelpFormatter();
final String cmdLineSyntax = formatCommandLineSyntax();
formatter.printHelp(printWriter,
80,
cmdLineSyntax,
null,
COMPLETE_OPTIONS,
1,
2,
null,
false);
output.append(stringWriter);
}
private static String formatCommandLineSyntax() {
final StringBuilder cmdLineSyntax = new StringBuilder();
cmdLineSyntax.append(LIBRARY_NAME);
formatOptionGroup(cmdLineSyntax);
formatOptions(cmdLineSyntax, REQUIRED_OPTIONS);
formatOptions(cmdLineSyntax, OPTIONAL_OPTIONS);
return cmdLineSyntax.toString();
}
private static void formatOptions(
final StringBuilder cmdLineSyntax,
final Collection