...
增加配置，这里在命令行做了两件事情：a. 在es服务上面安装s3-repository插件，b. 将aws s3的访问信息加入到es库中
代码块 # elasticsearch-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: name: elasticsearch spec: replicas: 1 selector: matchLabels: app: elasticsearch template: metadata: labels: app: elasticsearch spec: ... ################## # 新增部分 lifecycle: postStart: exec: command: ["/bin/bash", "-c", "/usr/share/elasticsearch/bin/elasticsearch-plugin list | grep -q repository-s3 || /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch repository-s3 && \ echo ${S3_CLIENT_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.access_key --stdin &&\ echo ${S3_CLIENT_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.secret_key --stdin"] ################## ... --- # Elasticsearch Configuration apiVersion: v1 kind: ConfigMap metadata: name: elasticsearch-config data: elasticsearch.yml: | ... ################## # 新增部分 s3.client.default.endpoint: "s3.ap-northeast-1.amazonaws.com" s3.client.default.protocol: https s3.client.default.read_timeout: 50s s3.client.default.max_retries: 3 s3.client.default.use_throttle_retries: true ################## ...启动后在目标集群的kibana devtool中创建挂载的快照库
代码块 PUT _snapshot/s3_backup_repository { "type": "s3", "settings": { "region": "ap-northeast-1", "bucket": "elasticserch-snapshot-backup", "compress": true, "server_side_encryption": true, "storage_class": "standard" } } # 若成功，表明s3的快照库关联成功 { "acknowledge": true } # 若失败，则需要检查上一步中的S3_CLIENT_ACCESS_KEY和S3_CLIENT_SECRET_KEY是否成功添加 { "error" : { "root_cause" : [ { "type" : "repository_exception", "reason" : "[s3_backup_repository] Could not determine repository generation from root blobs" } ], "type" : "repository_exception", "reason" : "[s3_backup_repository] Could not determine repository generation from root blobs", "caused_by" : { "type" : "i_o_exception", "reason" : "Exception when listing blobs by prefix [index-]", "caused_by" : { "type" : "sdk_client_exception", "reason" : "The requested metadata is not found at http://169.254.169.254/latest/meta-data/iam/security-credentials/ " } } }, "status" : 500 }检查一下快照库中的快照是否存在，如果存在说明挂载的快照可用，此时就可以进行数据同步了
代码块 language json GET _snapshot/s3_backup_repository/_all { "snapshots" : [ { "snapshot" : "snapshot-20240917", "uuid" : "TVlHLRoMSXupw60xQgsWcA", "repository" : "s3_backup_repository", "version_id" : 7100299, "version" : "7.10.2", "indices" : [ "halley.0727.transfer_journal", "vega.0727.block_ids", "vega.0727.txn_events", "vega.0727.dag_inspector_block", "vega.0727.pending_txns", "halley.0727.block_ids", ".opendistro-anomaly-detector-jobs", "halley.0727.token_info", "barnard.0727.blocks", ".tasks", "proxima.0727.pending_txns", "barnard.0727.txn_events", "vega.0727.dag_inspector_height_group", "main.0727.market_cap", "barnard.0727.txn_infos", "txn_infos", "barnard.0727.market_cap_bak", "opendistro-sample-http-responses", "halley.0727.txn_events", "main.0727.pending_txns", "vega.0727.txn_infos", "proxima.0727.transfer_journal", "proxima.0727.address_holder", "halley.0727.txn_infos", "barnard.0727.txn_payloads", "vega.0727.transfer_journal", "barnard.0727.918.address_holder", ".opendistro-anomaly-detectors", "barnard.0914.txn_infos", ".opendistro-reports-definitions", ".opendistro_security", "main.0727.txn_payloads", "main.0727.token_info", ".opendistro-job-scheduler-lock", "halley.0727.txn_payloads", "main.0727.txn_infos", ".opendistro-anomaly-results-history-2021.05.07-1", "proxima.0727.token_info", "barnard.0727.market_cap", ".opendistro-reports-instances", "barnard.0727.block_ids", "main.0727.transfer_journal", "halley.0727.transfer", "vega.0727.txn_payloads", "halley.0727.address_holder", "vega.0727.market_cap", "proxima.0727.transfer", "vega.0727.uncle_blocks", "vega.0727.address_holder", ".opendistro-anomaly-checkpoints", "vega.0727.token_info", "halley.0727.blocks", "barnard.0727.txn_infos_0915", "main.0727.transfer", "halley.0727.uncle_blocks", ".kibana_1", "barnard.0727.address_holder", "proxima.0727.txn_infos", "proxima.0727.blocks", "halley.0727.market_cap", "proxima.0727.uncle_blocks", "barnard.0727.transfer_journal", "barnard.0727.token_info", "main.0727.uncle_blocks", "barnard.0727.uncle_blocks", "main.0727.block_ids", "vega.0727.blocks", "proxima.0727.market_cap", "barnard.0401.txn_infos", "halley.0727.pending_txns", ".opendistro-anomaly-detection-state", "vega.0727.transfer", "proxima.0727.txn_payloads", "barnard.0727.pending_txns", "main.0727.txn_events", "test_index", "main.0727.blocks", "barnard.0727.transfer", "proxima.0727.block_ids", "main.0727.address_holder", ".kibana_-1666338091_elastic_1", "vega.0727.dag_inspector_edge", "proxima.0727.txn_events" ], "data_streams" : [ ], "include_global_state" : true, "state" : "SUCCESS", "start_time" : "2024-09-17T05:04:52.562Z", "start_time_in_millis" : 1726549492562, "end_time" : "2024-09-17T07:08:33.370Z", "end_time_in_millis" : 1726556913370, "duration_in_millis" : 7420808, "failures" : [ ], "shards" : { "total" : 381, "failed" : 0, "successful" : 381 }, "feature_states" : [ ] } ], "total" : 1, "remaining" : 0 }
3. 在目标集群上同步挂载快照的数据
执行以下命令进行同步
代码块 language json POST _snapshot/s3_backup_repository/snapshot-20240917/_restore { "indices": "barnard.0727.uncle_blocks", "ignore_unavailable": true, "include_global_state": false } # 成功 { "acknowledge": true }查看同步进度，可以通过检查doc的数量与源集群的索引是否一致来判断是否同步完成
代码块 language json GET /barnard.0727.uncle_blocks/_count { "count" : 1109354, "_shards" : { "total" : 5, "successful" : 5, "skipped" : 0, "failed" : 0 } }