Criar ClickPipe

Este endpoint está em beta. O contrato da API é estável, e não são esperadas alterações incompatíveis no futuro.

Cria um novo ClickPipe.

POST /v1/organizations/{organizationId}/services/{serviceId}/clickpipes

Criar ClickPipe

cURL

curl --request POST \
  --url https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes \
  --header 'Authorization: Basic <encoded-value>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "destination": {
    "columns": [
      {
        "name": "<string>",
        "type": "<string>"
      }
    ],
    "database": "<string>",
    "managedTable": true,
    "roles": [
      "<string>"
    ],
    "table": "<string>",
    "tableDefinition": {
      "engine": {
        "columnIds": [
          "<string>"
        ],
        "versionColumnId": "<string>"
      },
      "partitionBy": "<string>",
      "primaryKey": "<string>",
      "sortingKey": [
        "<string>"
      ]
    }
  },
  "fieldMappings": [
    {
      "destinationField": "<string>",
      "sourceField": "<string>"
    }
  ],
  "name": "<string>",
  "scaling": {
    "concurrency": 123,
    "replicaCpuMillicores": 1062,
    "replicaMemoryGb": 4.25,
    "replicas": 123
  },
  "settings": {
    "clickhouse_max_download_threads": 4,
    "clickhouse_max_insert_threads": 1,
    "clickhouse_max_threads": 8,
    "clickhouse_min_insert_block_size_bytes": 1073741824,
    "clickhouse_parallel_distributed_insert_select": 2,
    "clickhouse_parallel_view_processing": false,
    "object_storage_concurrency": 1,
    "object_storage_max_file_count": 100,
    "object_storage_max_insert_bytes": 10737418240,
    "object_storage_polling_interval_ms": 30000,
    "object_storage_use_cluster_function": true,
    "streaming_max_insert_wait_ms": 5000
  },
  "source": {
    "kafka": {
      "brokers": "<string>",
      "caCertificate": "<string>",
      "consumerGroup": "my-clickpipe-consumer-group",
      "credentials": {
        "password": "<string>",
        "username": "<string>"
      },
      "iamRole": "arn:aws:iam::123456789012:role/MyRole",
      "offset": {
        "timestamp": "2021-01-01T00:00"
      },
      "reversePrivateEndpointIds": [
        "<string>"
      ],
      "schemaRegistry": {
        "authentication": "PLAIN",
        "caCertificate": "<string>",
        "credentials": {
          "password": "<string>",
          "username": "<string>"
        },
        "url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004"
      },
      "topics": "<string>"
    },
    "kinesis": {
      "accessKey": {
        "accessKeyId": "<string>",
        "secretKey": "<string>"
      },
      "iamRole": "arn:aws:iam::123456789012:role/MyRole",
      "region": "us-east-1",
      "streamName": "my-stream",
      "timestamp": 1615766400,
      "useEnhancedFanOut": true
    },
    "objectStorage": {
      "accessKey": {
        "accessKeyId": "<string>",
        "secretKey": "<string>"
      },
      "azureContainerName": "mycontainer",
      "compression": "auto",
      "connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
      "delimiter": ",",
      "iamRole": "arn:aws:iam::123456789012:role/MyRole",
      "isContinuous": true,
      "path": "data/logs/*.json",
      "queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
      "url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz"
    },
    "postgres": {
      "credentials": {
        "password": "<string>",
        "username": "<string>"
      },
      "database": "<string>",
      "host": "<string>",
      "port": 123,
      "settings": {
        "allowNullableColumns": true,
        "enableFailoverSlots": true,
        "initialLoadParallelism": 123,
        "publicationName": "<string>",
        "pullBatchSize": 123,
        "replicationSlotName": "<string>",
        "snapshotNumRowsPerPartition": 123,
        "snapshotNumberOfParallelTables": 123,
        "syncIntervalSeconds": 123
      },
      "tableMappings": [
        {
          "excludedColumns": [
            "<string>"
          ],
          "sortingKeys": [
            "<string>"
          ],
          "sourceSchemaName": "<string>",
          "sourceTable": "<string>",
          "targetTable": "<string>",
          "useCustomSortingKey": true
        }
      ]
    },
    "validateSamples": true
  }
}
'

200

{
  "requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "result": {
    "createdAt": "<string>",
    "destination": {
      "columns": [
        {
          "name": "<string>",
          "type": "<string>"
        }
      ],
      "database": "<string>",
      "managedTable": true,
      "table": "<string>",
      "tableDefinition": {
        "engine": {
          "columnIds": [
            "<string>"
          ],
          "versionColumnId": "<string>"
        },
        "partitionBy": "<string>",
        "primaryKey": "<string>",
        "sortingKey": [
          "<string>"
        ]
      }
    },
    "fieldMappings": [
      {
        "destinationField": "<string>",
        "sourceField": "<string>"
      }
    ],
    "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
    "name": "<string>",
    "scaling": {
      "concurrency": 123,
      "replicaCpuMillicores": 1062,
      "replicaMemoryGb": 4.25,
      "replicas": 123
    },
    "serviceId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
    "settings": {
      "clickhouse_max_download_threads": 4,
      "clickhouse_max_insert_threads": 1,
      "clickhouse_max_threads": 8,
      "clickhouse_min_insert_block_size_bytes": 1073741824,
      "clickhouse_parallel_distributed_insert_select": 2,
      "clickhouse_parallel_view_processing": false,
      "object_storage_concurrency": 1,
      "object_storage_max_file_count": 100,
      "object_storage_max_insert_bytes": 10737418240,
      "object_storage_polling_interval_ms": 30000,
      "object_storage_use_cluster_function": true,
      "streaming_max_insert_wait_ms": 5000
    },
    "source": {
      "kafka": {
        "brokers": "<string>",
        "caCertificate": "<string>",
        "consumerGroup": "my-clickpipe-consumer-group",
        "iamRole": "arn:aws:iam::123456789012:role/MyRole",
        "offset": {
          "timestamp": "2021-01-01T00:00"
        },
        "reversePrivateEndpointIds": [
          "<string>"
        ],
        "schemaRegistry": {
          "authentication": "PLAIN",
          "caCertificate": "<string>",
          "url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004"
        },
        "topics": "<string>"
      },
      "kinesis": {
        "iamRole": "arn:aws:iam::123456789012:role/MyRole",
        "region": "us-east-1",
        "streamName": "my-stream",
        "timestamp": 1615766400,
        "useEnhancedFanOut": true
      },
      "objectStorage": {
        "azureContainerName": "mycontainer",
        "compression": "auto",
        "connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
        "delimiter": ",",
        "iamRole": "arn:aws:iam::123456789012:role/MyRole",
        "isContinuous": true,
        "path": "data/logs/*.json",
        "queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
        "url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz"
      },
      "postgres": {
        "database": "<string>",
        "host": "<string>",
        "port": 123,
        "settings": {
          "allowNullableColumns": true,
          "enableFailoverSlots": true,
          "initialLoadParallelism": 123,
          "publicationName": "<string>",
          "pullBatchSize": 123,
          "replicationSlotName": "<string>",
          "snapshotNumRowsPerPartition": 123,
          "snapshotNumberOfParallelTables": 123,
          "syncIntervalSeconds": 123
        },
        "tableMappings": [
          {
            "excludedColumns": [
              "<string>"
            ],
            "sortingKeys": [
              "<string>"
            ],
            "sourceSchemaName": "<string>",
            "sourceTable": "<string>",
            "targetTable": "<string>",
            "useCustomSortingKey": true
          }
        ]
      }
    },
    "state": "<string>",
    "updatedAt": "<string>"
  },
  "status": 200
}

Autorizações

Authorization · string · header · obrigatório

Use o ID da chave e o segredo da chave obtidos no ClickHouse Cloud console: https://clickhouse.com/docs/cloud/manage/openapi

Parâmetros de caminho

organizationId · string<uuid> · obrigatório

ID da organização proprietária do serviço.

serviceId · string<uuid> · obrigatório

ID do serviço para o qual o ClickPipe será criado.

Corpo

application/json

destination · object

Show child attributes

fieldMappings · object[]

Mapeamentos de campos do ClickPipe. Observe que todas as colunas de destino devem ser incluídas nos mapeamentos.

Show child attributes

name · string

Nome do ClickPipe.

scaling · object

Show child attributes

settings · object

Show child attributes

source · object

Show child attributes

Resposta

200

application/json

Resposta bem-sucedida

requestId · string<uuid>

ID único atribuído a cada solicitação. UUIDv4

result · object

Show child attributes

status · number

Código de status HTTP.

Exemplo:

200