mirror of
https://github.com/imjasonh/gcloud-help
synced 2026-07-09 19:06:53 +00:00
677 lines
24 KiB
Text
677 lines
24 KiB
Text
NAME
|
|
gcloud beta datastream streams create - creates a Datastream stream
|
|
|
|
SYNOPSIS
|
|
gcloud beta datastream streams create (STREAM : --location=LOCATION)
|
|
--display-name=DISPLAY_NAME
|
|
(--backfill-none
|
|
| --backfill-all --mongodb-excluded-objects=MONGODB_EXCLUDED_OBJECTS
|
|
| --mysql-excluded-objects=MYSQL_EXCLUDED_OBJECTS
|
|
| --oracle-excluded-objects=ORACLE_EXCLUDED_OBJECTS
|
|
| --postgresql-excluded-objects=POSTGRESQL_EXCLUDED_OBJECTS
|
|
| --salesforce-excluded-objects=SALESFORCE_EXCLUDED_OBJECTS
|
|
| --spanner-excluded-objects=SPANNER_EXCLUDED_OBJECTS
|
|
| --sqlserver-excluded-objects=SQLSERVER_EXCLUDED_OBJECTS)
|
|
(--destination-name=DESTINATION_NAME
|
|
(--bigquery-destination-config=BIGQUERY_DESTINATION_CONFIG
|
|
| --gcs-destination-config=GCS_DESTINATION_CONFIG))
|
|
(--source-name=SOURCE_NAME
|
|
(--mongodb-source-config=MONGODB_SOURCE_CONFIG
|
|
| --mysql-source-config=MYSQL_SOURCE_CONFIG
|
|
| --oracle-source-config=ORACLE_SOURCE_CONFIG
|
|
| --postgresql-source-config=POSTGRESQL_SOURCE_CONFIG
|
|
| --salesforce-source-config=SALESFORCE_SOURCE_CONFIG
|
|
| --spanner-source-config=SPANNER_SOURCE_CONFIG
|
|
| --sqlserver-source-config=SQLSERVER_SOURCE_CONFIG))
|
|
[--labels=[KEY=VALUE,...]] [--rule-sets=RULE_SETS]
|
|
[--force | --validate-only] [GCLOUD_WIDE_FLAG ...]
|
|
|
|
DESCRIPTION
|
|
(BETA) (DEPRECATED) Datastream beta version is deprecated. Please
|
|
use`gcloud datastream streams create command instead.
|
|
|
|
Create a Datastream stream. If successful, the response body contains a
|
|
newly created instance of Operation. To get the operation result, call:
|
|
describe OPERATION
|
|
|
|
EXAMPLES
|
|
To create a stream with an Oracle source and a Google Cloud Storage
|
|
destination:
|
|
|
|
$ gcloud beta datastream streams create STREAM \
|
|
--location=us-central1 --display-name=my-stream \
|
|
--source-name=source --oracle-source-config=source_config.json \
|
|
--destination-name=destination \
|
|
--gcs-destination-config=destination_config.json --backfill-none
|
|
|
|
To create a stream with a MySQL source and a Cloud Storage destination and
|
|
that excludes some objects from being backfilled:
|
|
|
|
$ gcloud beta datastream streams create STREAM \
|
|
--location=us-central1 --display-name=my-stream \
|
|
--source-name=source --mysql-source-config=source_config.json \
|
|
--destination-name=destination \
|
|
--gcs-destination-config=destination_config.json \
|
|
--backfill-all --mysql-excluded-objects=excluded_objects.json
|
|
|
|
POSITIONAL ARGUMENTS
|
|
Stream resource - The stream to create. The arguments in this group can be
|
|
used to specify the attributes of this resource. (NOTE) Some attributes
|
|
are not given arguments in this group but can be set in other ways.
|
|
|
|
To set the project attribute:
|
|
◆ provide the argument stream on the command line with a fully
|
|
specified name;
|
|
◆ provide the argument --project on the command line;
|
|
◆ set the property core/project.
|
|
|
|
This must be specified.
|
|
|
|
STREAM
|
|
ID of the stream or fully qualified identifier for the stream.
|
|
|
|
To set the stream attribute:
|
|
▸ provide the argument stream on the command line.
|
|
|
|
This positional argument must be specified if any of the other
|
|
arguments in this group are specified.
|
|
|
|
--location=LOCATION
|
|
The Cloud location for the stream.
|
|
|
|
To set the location attribute:
|
|
▸ provide the argument stream on the command line with a fully
|
|
specified name;
|
|
▸ provide the argument --location on the command line.
|
|
|
|
REQUIRED FLAGS
|
|
--display-name=DISPLAY_NAME
|
|
Friendly name for the stream.
|
|
|
|
Exactly one of these must be specified:
|
|
|
|
--backfill-none
|
|
Do not automatically backfill any objects. This flag is equivalent to
|
|
selecting the Manual backfill type in the Google Cloud console.
|
|
|
|
Or at least one of these can be specified:
|
|
|
|
--backfill-all
|
|
Automatically backfill objects included in the stream source
|
|
configuration. Specific objects can be excluded. This flag is
|
|
equivalent to selecting the Automatic backfill type in the Google
|
|
Cloud console.
|
|
|
|
At most one of these can be specified:
|
|
|
|
--mongodb-excluded-objects=MONGODB_EXCLUDED_OBJECTS
|
|
Path to a YAML (or JSON) file containing the MongoDB data sources
|
|
to avoid backfilling.
|
|
|
|
The JSON file is formatted as follows, with camelCase field
|
|
naming:
|
|
|
|
{
|
|
"databases": [
|
|
{
|
|
"database":"sample_database",
|
|
"collections": [
|
|
{
|
|
"collection": "sample_collection",
|
|
"fields": [
|
|
{
|
|
"field": "sample_field",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
--mysql-excluded-objects=MYSQL_EXCLUDED_OBJECTS
|
|
Path to a YAML (or JSON) file containing the MySQL data sources
|
|
to avoid backfilling.
|
|
|
|
The JSON file is formatted as follows, with camelCase field
|
|
naming:
|
|
|
|
{
|
|
"mysqlDatabases": [
|
|
{
|
|
"database":"sample_database",
|
|
"mysqlTables": [
|
|
{
|
|
"table": "sample_table",
|
|
"mysqlColumns": [
|
|
{
|
|
"column": "sample_column",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
--oracle-excluded-objects=ORACLE_EXCLUDED_OBJECTS
|
|
Path to a YAML (or JSON) file containing the Oracle data sources
|
|
to avoid backfilling.
|
|
|
|
The JSON file is formatted as follows, with camelCase field
|
|
naming:
|
|
|
|
{
|
|
"oracleSchemas": [
|
|
{
|
|
"schema": "SAMPLE",
|
|
"oracleTables": [
|
|
{
|
|
"table": "SAMPLE_TABLE",
|
|
"oracleColumns": [
|
|
{
|
|
"column": "COL",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
--postgresql-excluded-objects=POSTGRESQL_EXCLUDED_OBJECTS
|
|
Path to a YAML (or JSON) file containing the PostgreSQL data
|
|
sources to avoid backfilling.
|
|
|
|
The JSON file is formatted as follows, with camelCase field
|
|
naming:
|
|
|
|
{
|
|
"postgresqlSchemas": [
|
|
{
|
|
"schema": "SAMPLE",
|
|
"postgresqlTables": [
|
|
{
|
|
"table": "SAMPLE_TABLE",
|
|
"postgresqlColumns": [
|
|
{
|
|
"column": "COL",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
--salesforce-excluded-objects=SALESFORCE_EXCLUDED_OBJECTS
|
|
Path to a YAML (or JSON) file containing the Salesforce data
|
|
sources to avoid backfilling.
|
|
|
|
The JSON file is formatted as follows, with camelCase field
|
|
naming:
|
|
|
|
{
|
|
"objects": [
|
|
{
|
|
"objectName": "SAMPLE",
|
|
},
|
|
{
|
|
"objectName": "SAMPLE2",
|
|
}
|
|
]
|
|
}
|
|
|
|
--spanner-excluded-objects=SPANNER_EXCLUDED_OBJECTS
|
|
Path to a YAML (or JSON) file containing the Spanner data sources
|
|
to avoid backfilling.
|
|
|
|
The JSON file is formatted as follows, with camelCase field
|
|
naming:
|
|
|
|
{
|
|
"schemas": [
|
|
{
|
|
"schema": "SAMPLE_SCHEMA",
|
|
"tables": [
|
|
{
|
|
"table": "SAMPLE_TABLE",
|
|
"columns": [
|
|
{
|
|
"column": "SAMPLE_COLUMN",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
--sqlserver-excluded-objects=SQLSERVER_EXCLUDED_OBJECTS
|
|
Path to a YAML (or JSON) file containing the SQL Server data
|
|
sources to avoid backfilling.
|
|
|
|
The JSON file is formatted as follows, with camelCase field
|
|
naming:
|
|
|
|
{
|
|
"schemas": [
|
|
{
|
|
"schema": "SAMPLE",
|
|
"tables": [
|
|
{
|
|
"table": "SAMPLE_TABLE",
|
|
"columns": [
|
|
{
|
|
"column": "COL",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
This must be specified.
|
|
|
|
Connection profile resource - Resource ID of the destination connection
|
|
profile. This represents a Cloud resource. (NOTE) Some attributes are
|
|
not given arguments in this group but can be set in other ways.
|
|
|
|
To set the project attribute:
|
|
▸ provide the argument --destination-name on the command line with a
|
|
fully specified name;
|
|
▸ provide the argument --project on the command line;
|
|
▸ set the property core/project.
|
|
|
|
To set the location attribute:
|
|
▸ provide the argument --destination-name on the command line with a
|
|
fully specified name;
|
|
▸ provide the argument --location on the command line.
|
|
|
|
This must be specified.
|
|
|
|
--destination-name=DESTINATION_NAME
|
|
ID of the connection_profile or fully qualified identifier for the
|
|
connection_profile.
|
|
|
|
To set the connection_profile attribute:
|
|
▫ provide the argument --destination-name on the command line.
|
|
|
|
Exactly one of these must be specified:
|
|
|
|
--bigquery-destination-config=BIGQUERY_DESTINATION_CONFIG
|
|
Path to a YAML (or JSON) file containing the configuration for
|
|
Google BigQuery Destination Config.
|
|
|
|
The YAML (or JSON) file should be formatted as follows:
|
|
|
|
BigQuery configuration with source hierarchy datasets and merge
|
|
mode (merge mode is by default):
|
|
|
|
{
|
|
"sourceHierarchyDatasets": {
|
|
"datasetTemplate": {
|
|
"location": "us-central1",
|
|
"datasetIdPrefix": "my_prefix",
|
|
"kmsKeyName": "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{cryptoKey}"
|
|
}
|
|
},
|
|
"merge": {}
|
|
"dataFreshness": "3600s"
|
|
}
|
|
|
|
BigQuery configuration with source hierarchy datasets and append
|
|
only mode: {
|
|
"sourceHierarchyDatasets": {
|
|
"datasetTemplate": {
|
|
"location": "us-central1",
|
|
"datasetIdPrefix": "my_prefix",
|
|
"kmsKeyName": "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{cryptoKey}"
|
|
}
|
|
},
|
|
"appendOnly": {}
|
|
}
|
|
|
|
BigQuery configuration with single target dataset and merge mode:
|
|
|
|
{
|
|
"singleTargetDataset": {
|
|
"datasetId": "projectId:my_dataset"
|
|
},
|
|
"merge": {}
|
|
"dataFreshness": "3600s"
|
|
}
|
|
|
|
BigQuery configuration with Big Lake table configuration: {
|
|
"singleTargetDataset": {
|
|
"datasetId": "projectId:datasetId"
|
|
},
|
|
"appendOnly": {},
|
|
"blmtConfig": {
|
|
"bucket": "bucketName",
|
|
"tableFormat": "ICEBERG",
|
|
"fileFormat": "PARQUET",
|
|
"connectionName": "projectId.region.connectionName",
|
|
"rootPath": "/root"
|
|
}
|
|
}
|
|
|
|
--gcs-destination-config=GCS_DESTINATION_CONFIG
|
|
Path to a YAML (or JSON) file containing the configuration for
|
|
Google Cloud Storage Destination Config.
|
|
|
|
The JSON file is formatted as follows:
|
|
|
|
{
|
|
"path": "some/path",
|
|
"fileRotationMb":5,
|
|
"fileRotationInterval":"15s",
|
|
"avroFileFormat": {}
|
|
}
|
|
|
|
This must be specified.
|
|
|
|
Connection profile resource - Resource ID of the source connection
|
|
profile. This represents a Cloud resource. (NOTE) Some attributes are
|
|
not given arguments in this group but can be set in other ways.
|
|
|
|
To set the project attribute:
|
|
▸ provide the argument --source-name on the command line with a fully
|
|
specified name;
|
|
▸ provide the argument --project on the command line;
|
|
▸ set the property core/project.
|
|
|
|
To set the location attribute:
|
|
▸ provide the argument --source-name on the command line with a fully
|
|
specified name;
|
|
▸ provide the argument --location on the command line.
|
|
|
|
This must be specified.
|
|
|
|
--source-name=SOURCE_NAME
|
|
ID of the connection_profile or fully qualified identifier for the
|
|
connection_profile.
|
|
|
|
To set the connection_profile attribute:
|
|
▫ provide the argument --source-name on the command line.
|
|
|
|
Exactly one of these must be specified:
|
|
|
|
--mongodb-source-config=MONGODB_SOURCE_CONFIG
|
|
Path to a YAML (or JSON) file containing the configuration for
|
|
MongoDB Source Config.
|
|
|
|
The JSON file is formatted as follows, with snake_case field
|
|
naming:
|
|
|
|
{
|
|
"includeObjects": {},
|
|
"excludeObjects": {
|
|
"databases": [
|
|
{
|
|
"database": "sampleDb",
|
|
"collections": [
|
|
{
|
|
"collection": "sampleCollection",
|
|
"fields": [
|
|
{
|
|
"field": "SAMPLE_FIELD",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
|
|
--mysql-source-config=MYSQL_SOURCE_CONFIG
|
|
Path to a YAML (or JSON) file containing the configuration for
|
|
MySQL Source Config.
|
|
|
|
The JSON file is formatted as follows, with snake_case field
|
|
naming:
|
|
|
|
{
|
|
"allowlist": {},
|
|
"rejectlist": {
|
|
"mysql_databases": [
|
|
{
|
|
"database_name":"sample_database",
|
|
"mysql_tables": [
|
|
{
|
|
"table_name": "sample_table",
|
|
"mysql_columns": [
|
|
{
|
|
"column_name": "sample_column",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
|
|
--oracle-source-config=ORACLE_SOURCE_CONFIG
|
|
Path to a YAML (or JSON) file containing the configuration for
|
|
Oracle Source Config.
|
|
|
|
The JSON file is formatted as follows, with snake_case field
|
|
naming:
|
|
|
|
{
|
|
"allowlist": {},
|
|
"rejectlist": {
|
|
"oracle_schemas": [
|
|
{
|
|
"schema_name": "SAMPLE",
|
|
"oracle_tables": [
|
|
{
|
|
"table_name": "SAMPLE_TABLE",
|
|
"oracle_columns": [
|
|
{
|
|
"column_name": "COL",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
|
|
--postgresql-source-config=POSTGRESQL_SOURCE_CONFIG
|
|
Path to a YAML (or JSON) file containing the configuration for
|
|
PostgreSQL Source Config.
|
|
|
|
The JSON file is formatted as follows, with camelCase field naming:
|
|
|
|
{
|
|
"includeObjects": {},
|
|
"excludeObjects": {
|
|
"postgresqlSchemas": [
|
|
{
|
|
"schema": "SAMPLE",
|
|
"postgresqlTables": [
|
|
{
|
|
"table": "SAMPLE_TABLE",
|
|
"postgresqlColumns": [
|
|
{
|
|
"column": "COL",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"replicationSlot": "SAMPLE_REPLICATION_SLOT",
|
|
"publication": "SAMPLE_PUBLICATION"
|
|
}
|
|
|
|
--salesforce-source-config=SALESFORCE_SOURCE_CONFIG
|
|
Path to a YAML (or JSON) file containing the configuration for
|
|
Salesforce Source Config.
|
|
|
|
The JSON file is formatted as follows, with camelCase field naming:
|
|
|
|
{
|
|
"pollingInterval": "3000s",
|
|
"includeObjects": {},
|
|
"excludeObjects": {
|
|
"objects": [
|
|
{
|
|
"objectName": "SAMPLE",
|
|
"fields": [
|
|
{
|
|
"fieldName": "SAMPLE_FIELD",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
|
|
--spanner-source-config=SPANNER_SOURCE_CONFIG
|
|
Path to a YAML (or JSON) file containing the configuration for
|
|
Spanner Source Config.
|
|
|
|
The JSON file is formatted as follows, with camelCase field naming:
|
|
|
|
"changeStreamName": "SAMPLE_CHANGE_STREAM_NAME",
|
|
{
|
|
"includeObjects": {},
|
|
"excludeObjects": {
|
|
"schemas": [
|
|
{
|
|
"schema": "SAMPLE",
|
|
"tables": [
|
|
{
|
|
"table": "SAMPLE_TABLE",
|
|
"columns": [
|
|
{
|
|
"column": "COL",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"maxConcurrentCdcTasks": 1000,
|
|
"maxConcurrentBackfillTasks": 10,
|
|
"backfillDataBoostEnabled": false,
|
|
"fgacRole": "SAMPLE_FGAC_ROLE",
|
|
"spannerRpcPriority": "MEDIUM"
|
|
}
|
|
|
|
--sqlserver-source-config=SQLSERVER_SOURCE_CONFIG
|
|
Path to a YAML (or JSON) file containing the configuration for SQL
|
|
Server Source Config.
|
|
|
|
The JSON file is formatted as follows, with camelCase field naming:
|
|
|
|
{
|
|
"includeObjects": {},
|
|
"excludeObjects": {
|
|
"schemas": [
|
|
{
|
|
"schema": "SAMPLE",
|
|
"tables": [
|
|
{
|
|
"table": "SAMPLE_TABLE",
|
|
"columns": [
|
|
{
|
|
"column": "COL",
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"maxConcurrentCdcTasks": 2,
|
|
"maxConcurrentBackfillTasks": 10,
|
|
"transactionLogs": {} # Or changeTables
|
|
}
|
|
|
|
OPTIONAL FLAGS
|
|
--labels=[KEY=VALUE,...]
|
|
List of label KEY=VALUE pairs to add.
|
|
|
|
Keys must start with a lowercase character and contain only hyphens
|
|
(-), underscores (_), lowercase characters, and numbers. Values must
|
|
contain only hyphens (-), underscores (_), lowercase characters, and
|
|
numbers.
|
|
|
|
--rule-sets=RULE_SETS
|
|
Path to a JSON file containing a list of rule sets to be applied to the
|
|
stream.
|
|
|
|
The JSON file is formatted as follows, with camelCase field naming:
|
|
|
|
[
|
|
{
|
|
"objectFilter": {
|
|
"sourceObjectIdentifier": {
|
|
"oracleIdentifier": {
|
|
"schema": "schema1",
|
|
"table": "table1"
|
|
}
|
|
}
|
|
},
|
|
"customizationRules": [
|
|
{
|
|
"bigqueryClustering": {
|
|
"columns": ["COL_A"]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"objectFilter": {
|
|
"sourceObjectIdentifier": {
|
|
"oracleIdentifier": {
|
|
"schema": "schema2",
|
|
"table": "table2"
|
|
}
|
|
}
|
|
},
|
|
"customizationRules": [
|
|
{
|
|
"bigqueryPartitioning": {
|
|
"timeUnitPartition": {
|
|
"column": "TIME_COL",
|
|
"partitioningTimeGranularity": "PARTITIONING_TIME_GRANULARITY_DAY"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
|
|
At most one of these can be specified:
|
|
|
|
--force
|
|
Create the stream without validating it.
|
|
|
|
--validate-only
|
|
Only validate the stream, but do not create any resources. The
|
|
default is false.
|
|
|
|
GCLOUD WIDE FLAGS
|
|
These flags are available to all commands: --access-token-file, --account,
|
|
--billing-project, --configuration, --flags-file, --flatten, --format,
|
|
--help, --impersonate-service-account, --log-http, --project, --quiet,
|
|
--trace-token, --user-output-enabled, --verbosity.
|
|
|
|
Run $ gcloud help for details.
|
|
|
|
NOTES
|
|
This command is currently in beta and might change without notice. This
|
|
variant is also available:
|
|
|
|
$ gcloud datastream streams create
|
|
|