mirror of
https://github.com/imjasonh/gcloud-help
synced 2026-07-16 20:36:39 +00:00
gcloud: Wed Oct 5 10:40:53 UTC 2022
This commit is contained in:
Automated
parent
ffb9e43995
commit
344952a6dd
135 changed files with 3530 additions and 534 deletions
|
|
@ -7,7 +7,16 @@ SYNOPSIS
|
|||
: --execution-args=EXECUTION_ARGS
|
||||
--execution-project=EXECUTION_PROJECT --kms-key=KMS_KEY
|
||||
--max-job-execution-lifetime=MAX_JOB_EXECUTION_LIFETIME)
|
||||
((--spark-main-class=SPARK_MAIN_CLASS
|
||||
([--notebook=NOTEBOOK
|
||||
: --notebook-archive-uris=[NOTEBOOK_ARCHIVE_URIS,...]
|
||||
--notebook-file-uris=[NOTEBOOK_FILE_URIS,...]
|
||||
--notebook-batch-executors-count=NOTEBOOK_BATCH_EXECUTORS_COUNT
|
||||
--notebook-batch-max-executors-count=NOTEBOOK_BATCH_MAX_EXECUTORS_COUNT --notebook-container-image=NOTEBOOK_CONTAINER_IMAGE --notebook-container-image-java-jars=[NOTEBOOK_CONTAINER_IMAGE_JAVA_JARS,
|
||||
...]
|
||||
--notebook-container-image-properties=NOTEBOOK_CONTAINER_IMAGE_PROPERTIES --notebook-vpc-network-tags=[NOTEBOOK_VPC_NETWORK_TAGS,
|
||||
...] --notebook-vpc-network-name=NOTEBOOK_VPC_NETWORK_NAME
|
||||
| --notebook-vpc-sub-network-name=NOTEBOOK_VPC_SUB_NETWORK_NAME]
|
||||
| [(--spark-main-class=SPARK_MAIN_CLASS
|
||||
| --spark-main-jar-file-uri=SPARK_MAIN_JAR_FILE_URI
|
||||
| --spark-python-script-file=SPARK_PYTHON_SCRIPT_FILE
|
||||
| --spark-sql-script=SPARK_SQL_SCRIPT
|
||||
|
|
@ -22,7 +31,7 @@ SYNOPSIS
|
|||
--container-image-python-packages=[CONTAINER_IMAGE_PYTHON_PACKAGES,
|
||||
...] --vpc-network-tags=[VPC_NETWORK_TAGS,...]
|
||||
--vpc-network-name=VPC_NETWORK_NAME
|
||||
| --vpc-sub-network-name=VPC_SUB_NETWORK_NAME)
|
||||
| --vpc-sub-network-name=VPC_SUB_NETWORK_NAME])
|
||||
(--trigger-type=TRIGGER_TYPE : --trigger-disabled
|
||||
--trigger-max-retires=TRIGGER_MAX_RETIRES
|
||||
--trigger-schedule=TRIGGER_SCHEDULE
|
||||
|
|
@ -142,91 +151,158 @@ REQUIRED FLAGS
|
|||
--max-job-execution-lifetime=MAX_JOB_EXECUTION_LIFETIME
|
||||
The maximum duration before the job execution expires.
|
||||
|
||||
Config related to running custom Spark tasks.
|
||||
|
||||
At least one of these must be specified:
|
||||
|
||||
--spark-archive-uris=[SPARK_ARCHIVE_URIS,...]
|
||||
Google Cloud Storage URIs of archives to be extracted into the
|
||||
working directory of each executor. Supported file types: .jar, .tar,
|
||||
.tar.gz, .tgz, and .zip.
|
||||
|
||||
--spark-file-uris=[SPARK_FILE_URIS,...]
|
||||
Google Cloud Storage URIs of files to be placed in the working
|
||||
directory of each executor.
|
||||
|
||||
The specification of the main method to call to drive the job. Specify
|
||||
either the jar file that contains the main class or the main class name.
|
||||
Select which task you want to schedule and provide the required arguments
|
||||
for the task. The 2 types of tasks supported are:-
|
||||
◆ spark tasks
|
||||
◆ notebook tasks
|
||||
|
||||
Exactly one of these must be specified:
|
||||
|
||||
--spark-main-class=SPARK_MAIN_CLASS
|
||||
The name of the driver's main class. The jar file that contains
|
||||
the class must be in the default CLASSPATH or specified in
|
||||
jar_file_uris. The execution args are passed in as a sequence of
|
||||
named process arguments (--key=value).
|
||||
Config related to running custom notebook tasks.
|
||||
|
||||
--spark-main-jar-file-uri=SPARK_MAIN_JAR_FILE_URI
|
||||
The Google Cloud Storage URI of the jar file that contains the
|
||||
main class. The execution args are passed in as a sequence of
|
||||
named process arguments (--key=value).
|
||||
--notebook=NOTEBOOK
|
||||
Path to input notebook. This can be the Google Cloud Storage URI of
|
||||
the notebook file or the path to a Notebook Content. The execution
|
||||
args are accessible as environment variables (TASK_key=value).
|
||||
|
||||
--spark-python-script-file=SPARK_PYTHON_SCRIPT_FILE
|
||||
The Google Cloud Storage URI of the main Python file to use as
|
||||
the driver. Must be a .py file.
|
||||
This flag argument must be specified if any of the other arguments
|
||||
in this group are specified.
|
||||
|
||||
--spark-sql-script=SPARK_SQL_SCRIPT
|
||||
The SQL query text.
|
||||
--notebook-archive-uris=[NOTEBOOK_ARCHIVE_URIS,...]
|
||||
Google Cloud Storage URIs of archives to be extracted into the
|
||||
working directory of each executor. Supported file types: .jar,
|
||||
.tar, .tar.gz, .tgz, and .zip.
|
||||
|
||||
--spark-sql-script-file=SPARK_SQL_SCRIPT_FILE
|
||||
A reference to a query file. This can be the Google Cloud Storage
|
||||
URI of the query file or it can the path to a SqlScript Content.
|
||||
--notebook-file-uris=[NOTEBOOK_FILE_URIS,...]
|
||||
Google Cloud Storage URIs of files to be placed in the working
|
||||
directory of each executor.
|
||||
|
||||
Compute resources needed for a Task when using Dataproc Serverless.
|
||||
Compute resources needed for a Task when using Dataproc Serverless.
|
||||
|
||||
--batch-executors-count=BATCH_EXECUTORS_COUNT
|
||||
Total number of job executors.
|
||||
--notebook-batch-executors-count=NOTEBOOK_BATCH_EXECUTORS_COUNT
|
||||
Total number of job executors.
|
||||
|
||||
--batch-max-executors-count=BATCH_MAX_EXECUTORS_COUNT
|
||||
Max configurable executors. If max_executors_count >
|
||||
executors_count, then auto-scaling is enabled.
|
||||
--notebook-batch-max-executors-count=NOTEBOOK_BATCH_MAX_EXECUTORS_COUNT
|
||||
Max configurable executors. If max_executors_count >
|
||||
executors_count, then auto-scaling is enabled.
|
||||
|
||||
Container Image Runtime Configuration.
|
||||
Container Image Runtime Configuration.
|
||||
|
||||
--container-image=CONTAINER_IMAGE
|
||||
Optional custom container image for the job.
|
||||
--notebook-container-image=NOTEBOOK_CONTAINER_IMAGE
|
||||
Optional custom container image for the job.
|
||||
|
||||
--container-image-java-jars=[CONTAINER_IMAGE_JAVA_JARS,...]
|
||||
A list of Java JARS to add to the classpath. Valid input includes
|
||||
Cloud Storage URIs to Jar binaries. For example,
|
||||
gs://bucket-name/my/path/to/file.jar
|
||||
--notebook-container-image-java-jars=[NOTEBOOK_CONTAINER_IMAGE_JAVA_JARS,...]
|
||||
A list of Java JARS to add to the classpath. Valid input includes
|
||||
Cloud Storage URIs to Jar binaries. For example,
|
||||
gs://bucket-name/my/path/to/file.jar
|
||||
|
||||
--container-image-properties=CONTAINER_IMAGE_PROPERTIES
|
||||
The properties to set on daemon config files. Property keys are
|
||||
specified in prefix:property format, for example
|
||||
core:hadoop.tmp.dir. For more information, see Cluster properties
|
||||
(https://cloud.google.com/dataproc/docs/concepts/cluster-properties)
|
||||
--notebook-container-image-properties=NOTEBOOK_CONTAINER_IMAGE_PROPERTIES
|
||||
The properties to set on daemon config files. Property keys are
|
||||
specified in prefix:property format, for example
|
||||
core:hadoop.tmp.dir. For more information, see Cluster properties
|
||||
(https://cloud.google.com/dataproc/docs/concepts/cluster-properties)
|
||||
|
||||
--container-image-python-packages=[CONTAINER_IMAGE_PYTHON_PACKAGES,...]
|
||||
A list of python packages to be installed. Valid formats include
|
||||
Cloud Storage URI to a PIP installable library. For example,
|
||||
gs://bucket-name/my/path/to/lib.tar.gz
|
||||
Cloud VPC Network used to run the infrastructure.
|
||||
|
||||
Cloud VPC Network used to run the infrastructure.
|
||||
--notebook-vpc-network-tags=[NOTEBOOK_VPC_NETWORK_TAGS,...]
|
||||
List of network tags to apply to the job.
|
||||
|
||||
--vpc-network-tags=[VPC_NETWORK_TAGS,...]
|
||||
List of network tags to apply to the job.
|
||||
The Cloud VPC network identifier.
|
||||
|
||||
The Cloud VPC network identifier.
|
||||
At most one of these can be specified:
|
||||
|
||||
At most one of these can be specified:
|
||||
--notebook-vpc-network-name=NOTEBOOK_VPC_NETWORK_NAME
|
||||
The Cloud VPC network in which the job is run. By default, the
|
||||
Cloud VPC network named Default within the project is used.
|
||||
|
||||
--vpc-network-name=VPC_NETWORK_NAME
|
||||
The Cloud VPC network in which the job is run. By default, the
|
||||
Cloud VPC network named Default within the project is used.
|
||||
--notebook-vpc-sub-network-name=NOTEBOOK_VPC_SUB_NETWORK_NAME
|
||||
The Cloud VPC sub-network in which the job is run.
|
||||
|
||||
--vpc-sub-network-name=VPC_SUB_NETWORK_NAME
|
||||
The Cloud VPC sub-network in which the job is run.
|
||||
Config related to running custom Spark tasks.
|
||||
|
||||
--spark-archive-uris=[SPARK_ARCHIVE_URIS,...]
|
||||
Google Cloud Storage URIs of archives to be extracted into the
|
||||
working directory of each executor. Supported file types: .jar,
|
||||
.tar, .tar.gz, .tgz, and .zip.
|
||||
|
||||
--spark-file-uris=[SPARK_FILE_URIS,...]
|
||||
Google Cloud Storage URIs of files to be placed in the working
|
||||
directory of each executor.
|
||||
|
||||
The specification of the main method to call to drive the job. Specify
|
||||
either the jar file that contains the main class or the main class
|
||||
name.
|
||||
|
||||
Exactly one of these must be specified:
|
||||
|
||||
--spark-main-class=SPARK_MAIN_CLASS
|
||||
The name of the driver's main class. The jar file that contains
|
||||
the class must be in the default CLASSPATH or specified in
|
||||
jar_file_uris. The execution args are passed in as a sequence
|
||||
of named process arguments (--key=value).
|
||||
|
||||
--spark-main-jar-file-uri=SPARK_MAIN_JAR_FILE_URI
|
||||
The Google Cloud Storage URI of the jar file that contains the
|
||||
main class. The execution args are passed in as a sequence of
|
||||
named process arguments (--key=value).
|
||||
|
||||
--spark-python-script-file=SPARK_PYTHON_SCRIPT_FILE
|
||||
The Google Cloud Storage URI of the main Python file to use as
|
||||
the driver. Must be a .py file.
|
||||
|
||||
--spark-sql-script=SPARK_SQL_SCRIPT
|
||||
The SQL query text.
|
||||
|
||||
--spark-sql-script-file=SPARK_SQL_SCRIPT_FILE
|
||||
A reference to a query file. This can be the Google Cloud
|
||||
Storage URI of the query file or it can the path to a SqlScript
|
||||
Content.
|
||||
|
||||
Compute resources needed for a Task when using Dataproc Serverless.
|
||||
|
||||
--batch-executors-count=BATCH_EXECUTORS_COUNT
|
||||
Total number of job executors.
|
||||
|
||||
--batch-max-executors-count=BATCH_MAX_EXECUTORS_COUNT
|
||||
Max configurable executors. If max_executors_count >
|
||||
executors_count, then auto-scaling is enabled.
|
||||
|
||||
Container Image Runtime Configuration.
|
||||
|
||||
--container-image=CONTAINER_IMAGE
|
||||
Optional custom container image for the job.
|
||||
|
||||
--container-image-java-jars=[CONTAINER_IMAGE_JAVA_JARS,...]
|
||||
A list of Java JARS to add to the classpath. Valid input includes
|
||||
Cloud Storage URIs to Jar binaries. For example,
|
||||
gs://bucket-name/my/path/to/file.jar
|
||||
|
||||
--container-image-properties=CONTAINER_IMAGE_PROPERTIES
|
||||
The properties to set on daemon config files. Property keys are
|
||||
specified in prefix:property format, for example
|
||||
core:hadoop.tmp.dir. For more information, see Cluster properties
|
||||
(https://cloud.google.com/dataproc/docs/concepts/cluster-properties)
|
||||
|
||||
--container-image-python-packages=[CONTAINER_IMAGE_PYTHON_PACKAGES,...]
|
||||
A list of python packages to be installed. Valid formats include
|
||||
Cloud Storage URI to a PIP installable library. For example,
|
||||
gs://bucket-name/my/path/to/lib.tar.gz
|
||||
|
||||
Cloud VPC Network used to run the infrastructure.
|
||||
|
||||
--vpc-network-tags=[VPC_NETWORK_TAGS,...]
|
||||
List of network tags to apply to the job.
|
||||
|
||||
The Cloud VPC network identifier.
|
||||
|
||||
At most one of these can be specified:
|
||||
|
||||
--vpc-network-name=VPC_NETWORK_NAME
|
||||
The Cloud VPC network in which the job is run. By default, the
|
||||
Cloud VPC network named Default within the project is used.
|
||||
|
||||
--vpc-sub-network-name=VPC_SUB_NETWORK_NAME
|
||||
The Cloud VPC sub-network in which the job is run.
|
||||
|
||||
Spec related to Dataplex task scheduling and frequency settings.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue