# Technical Documentation: tencentmusic/cube-studio > ℹ️ **Provenance:** In-Tree Documentation (11 Chapters) · [CodeWiki Reference](https://codewiki.google/github.com/tencentmusic/cube-studio) · Recency: Active (< 180 days) ## File: README.md # 仓库已归档 本仓库已于 2026-05-18 归档，不再维护。 [跳转至新仓库](https://github.com/data-infra/cube-studio) # 帮助文档 参考 [帮助文档](https://github.com/data-infra/cube-studio/wiki) # Cube Studio ### 整体架构 --- ## File: install/README.md # 帮助文档 [帮助文档](https://github.com/data-infra/cube-studio/wiki) --- ## File: job-template/README.md # 帮助文档 [帮助文档](https://github.com/data-infra/cube-studio/wiki) --- ## File: myapp/init/init-automl.json [ { "job_type": "Job", "project": "public", "name": "test1", "namespace": "automl", "describe": "nni单机并行功能测试", "parallel_trial_count": 3, "max_trial_count": 12, "objective_type": "maximize", "objective_goal": 0.99, "objective_metric_name": "accuracy", "algorithm_name": "Random", "parameters": "{\n \"batch_size\": {\n \"_type\": \"choice\",\n \"_value\": [16,32,64,128]\n },\n \"momentum\": {\n \"_type\": \"uniform\",\n \"_value\": [0,1]\n }\n}", "job_json": { }, "job_worker_image": "ccr.ccs.tencentyun.com/cube-studio/nni:20240501", "working_dir": "/mnt/admin/pipeline/example/nni/", "job_worker_command": "python demo.py", "resource_memory": "1G", "resource_cpu": "1", "resource_gpu": "0" } ] --- ## File: myapp/init/init-etl-pipeline.json [ { "name": "dau", "describe": "dau计算", "config": { "alert_user": "admin" }, "workflow": "airflow", "dag_json": { "cos导入hdfs-1686184253953": { "label": "数据导入", "location": [ 304, 96 ], "color": { "color": "rgba(0,170,200,1)", "bg": "rgba(0,170,200,0.02)" }, "template": "cos导入hdfs", "template-group": "出库入库", "task-config": { "crontab": "1 1 * * *", "selfDepend": "单实例运行", "ResourceGroup": "default", "alert_user": "admin,", "timeout": "0", "retry": "0", "hdfsPath": "hdfs://xx/xxx", "cosPath": "/xx/${YYYYMMDD}.tar.gz", "ifNeedZip": "1", "label": "数据导入" }, "upstream": [], "task_id": 1 }, "hdfs入库至hive-1686184263002": { "label": "数据入库", "location": [ 304, 224 ], "color": { "color": "rgba(0,170,200,1)", "bg": "rgba(0,170,200,0.02)" }, "template": "hdfs入库至hive", "template-group": "出库入库", "task-config": { "crontab": "1 1 * * *", "selfDepend": "单实例运行", "ResourceGroup": "default", "alert_user": "admin,", "timeout": "0", "retry": "0", "charSet": "UTF-8", "databaseName": "", "tableName": "", "delimiter": "9", "failedOnZeroWrited": "1", "partitionType": "P_${YYYYMMDDHH}", "sourceFilePath": "", "sourceFileNames": "*", "sourceColumnNames": "", "targetColumnNames": "", "loadMode": "TRUNCATE", "label": "数据入库" }, "upstream": [ "cos导入hdfs-1686184253953" ], "task_id": 2 }, "SQL-1686184276800": { "label": "局部特征计算", "location": [ -16, 352 ], "color": { "color": "rgba(0,200,153,1)", "bg": "rgba(0,200,153,0.02)" }, "template": "SQL", "template-group": "数据计算", "task-config": { "crontab": "1 1 * * *", "selfDepend": "单实例运行", "ResourceGroup": "default", "alert_user": "admin,", "timeout": "0", "retry": "0", "filterSQL": "\n --库名，替换下面的demo_database\n use demo_database;\n\n --建表语句，替换下面的demo_table，修改字段。一定要加“if not exists”，这样使只在第一次运行时建表\n CREATE TABLE if not exists demo_table(\n qimei36 STRING COMMENT '唯一设备ID',\n userid_id STRING COMMENT '用户id（各app的用户id）',\n device_id STRING COMMENT '设备id（各app的device_id）',\n ftime INT COMMENT '数据分区时间 格式：yyyymmdd'\n )\n PARTITION BY LIST( ftime ) --定义分区字段，替换掉ftime。\n (\n PARTITION p_20220323 VALUES IN ( 20220323 ), --初始分区，分区名替换p_20220323，分区值替换20220323\n PARTITION default\n )\n STORED AS ORCFILE COMPRESS;\n\n -- 分区，根据时间参数新建分区。\n alter table demo_table drop partition (p_${YYYYMMDD});\n alter table demo_table add partition p_${YYYYMMDD} values in (${YYYYMMDD});\n\n -- 写入，用你的sql逻辑替换。\n insert table demo_table\n select * from other_db::other_table partition(p_${YYYYMMDD}) t;\n ", "special_para": "set hive.exec.parallel = true;set hive.execute.engine=spark;set hive.multi.join.use.hive=false;set hive.spark.failed.retry=false;", "label": "局部特征计算" }, "upstream": [ "hdfs入库至hive-1686184263002" ], "task_id": 3 }, "SparkScala-1686184279367": { "label": "局部特征计算", "location": [ 304, 352 ], "color": { "color": "rgba(0,200,153,1)", "bg": "rgba(0,200,153,0.02)" }, "template": "SparkScala", "template-group": "数据计算", "task-config": { "crontab": "1 1 * * *", "selfDepend": "单实例运行", "ResourceGroup": "default", "alert_user": "admin,", "timeout": "0", "retry": "0", "jar_path": "", "className": "", "files": "", "programSpecificParams": "", "options": "", "dynamicAllocation": "1", "driver_memory": "2g", "num_executors": "4", "executor_memory": "2g", "executor_cores": "2", "task.main.timeout": "480", "task.check.timeout": "5", "label": "局部特征计算" }, "upstream": [ "hdfs入库至hive-1686184263002" ], "task_id": 4 }, "pyspark-1686184281148": { "label": "局部特征计算", "location": [ 608, 352 ], "color": { "color": "rgba(0,200,153,1)", "bg": "rgba(0,200,153,0.02)" }, "template": "pyspark", "template-group": "数据计算", "task-config": { "crontab": "1 1 * * *", "selfDepend": "单实例运行", "ResourceGroup": "default", "alert_user": "admin,", "timeout": "0", "retry": "0", "py_script_path": "", "files": "", "pyFiles": "", "programSpecificParams": "", "options": "", "dynamicAllocation": 1, "driver_memory": "2g", "num_executors": 4, "executor_memory": "2g", "executor_cores": 2, "task.main.timeout": 480, "task.check.timeout": "5", "label": "局部特征计算" }, "upstream": [ "hdfs入库至hive-1686184263002" ], "task_id": 5 }, "hive出库至hdfs-1686184293917": { "label": "结果计算", "location": [ 304, 496 ], "color": { "color": "rgba(0,170,200,1)", "bg": "rgba(0,170,200,0.02)" }, "template": "hive出库至hdfs", "template-group": "出库入库", "task-config": { "crontab": "1 1 * * *", "selfDepend": "单实例运行", "ResourceGroup": "default", "alert_user": "admin,", "timeout": "0", "retry": "0", "databaseName": "", "destCheckFileName": "", "destCheckFilePath": "", "destFileDelimiter": "9", "destFilePath": "", "filterSQL": "select t1,t2,t3 from your_table where imp_date=${YYYYMMDD}", "label": "结果计算" }, "upstream": [ "SQL-1686184276800", "pyspark-1686184281148", "SparkScala-1686184279367" ], "task_id": 6 }, "hdfs导入cos-1686184296749": { "label": "数据导出", "location": [ 304, 608 ], "color": { "color": "rgba(0,170,200,1)", "bg": "rgba(0,170,200,0.02)" }, "template": "hdfs导入cos", "template-group": "出库入库", "task-config": { "crontab": "1 1 * * *", "selfDepend": "单实例运行", "ResourceGroup": "default", "alert_user": "admin,", "timeout": "0", "retry": "0", "hdfsPath": "hdfs://xx/xxx", "cosPath": "/xx/xx/${YYYYMMDD}.tar.gz", "ifNeedZip": "1", "label": "数据导出" }, "upstream": [ "hive出库至hdfs-1686184293917" ], "task_id": 7 } } } ] --- ## File: myapp/init/init-image.json [ { "group_name": "基础命令", "name": "ccr.ccs.tencentyun.com/cube-studio/python:strong", "repository": "hubsecret", "gitpath": "", "describe": "丰富版本python镜像", "dockerfile":"# 设置基础镜像\nFROM python:3.9\n# 配置环境变量\nENV TZ=Asia/Shanghai\nENV DEBIAN_FRONTEND=noninteractive\n# 配置源\nRUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple\n# 安装软件\nRUN pip install numpy \n# 设置工作目录\nWORKDIR /app\n# 拷贝文件，只能是/mnt/$username/下面的\nCOPY /mnt/admin/coco.zip /app/\n# 暴露端口\nEXPOSE 80\n# 设置启动命令\nENTRYPOINT [\"sleep\",\"100000\"]" } ] --- ## File: myapp/init/init-inference.json { "tf-mnist": { "project_name": "public", "service_name": "mnist-202208011", "model_name": "mnist", "service_describe": "tf 图像分类", "image_name": "ccr.ccs.tencentyun.com/cube-studio/tfserving:2.3.4", "model_version": "v2022.08.01.1", "model_path": "https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/tf-mnist.tar.gz", "service_type": "tfserving", "env": "TF_CPP_VMODULE=http_server=1\nTZ=Asia/Shanghai", "host": "/v1/models/mnist/metadata", "ports": "8501", "metrics": "8501:/metrics", "command": "wget https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/tf-mnist.tar.gz && tar -zxvf tf-mnist.tar.gz && mkdir -p /models/mnist/202207281/ && cp -r tf-mnist/* /models/mnist/202207281/ && /usr/bin/tf_serving_entrypoint.sh --model_config_file=/config/models.config --monitoring_config_file=/config/monitoring.config --platform_config_file=/config/platform.config --rest_api_num_threads=300 --enable_batching=true", "health": "8501:/v1/models/mnist/versions/202207281/metadata", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "expand": { } }, "pytorch-resnet50": { "project_name": "public", "service_name": "resnet50-202208012", "model_name": "resnet50", "service_describe": "pytorch 图像分类", "image_name": "ccr.ccs.tencentyun.com/cube-studio/torchserve:0.7.1-cpu", "model_version": "v2022.08.01.2", "model_path": "https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/resnet50.mar", "service_type": "torch-server", "env": "", "host": ":8081/models", "ports": "8080,8081", "metrics": "8082:/metrics", "workdir": "/models", "command": "wget https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/resnet50.mar && mkdir -p /models && cp /config/* /models/ && cp resnet50.mar /models/ && torchserve --start --model-store /models --models resnet50=resnet50.mar --foreground --ts-config=/config/config.properties", "health": "8080:/ping", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "5G", "resource_cpu": "5", "inference_config": "---config.properties\n\ninference_address=http://0.0.0.0:8080\nmanagement_address=http://0.0.0.0:8081\nmetrics_address=http://0.0.0.0:8082\ncors_allowed_origin=*\ncors_allowed_methods=GET, POST, PUT, OPTIONS\ncors_allowed_headers=X-Custom-Header\nnumber_of_netty_threads=32\nenable_metrics_api=true\njob_queue_size=1000\nenable_envvars_config=true\nasync_logging=true\ndefault_response_timeout=120\nmax_request_size=6553500\n", "expand": { } }, "torchscript-resnet50": { "project_name": "public", "service_name": "resnet50-202208013", "model_name": "resnet50", "service_describe": "torchscript 图像分类", "image_name": "ccr.ccs.tencentyun.com/cube-studio/tritonserver:22.07-py3", "model_version": "v2022.08.01.3", "model_path": "https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/resnet50-torchscript.pt", "service_type": "triton-server", "env": "", "host": "/v2/models/resnet50", "ports": "8000,8002", "metrics": "", "workdir": "", "command": "wget https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/resnet50-torchscript.pt && mkdir -p /models/resnet50/202208013/ && cp /config/* /models/resnet50/ && cp -r resnet50-torchscript.pt /models/resnet50/202208013/model.pt && tritonserver --model-repository=/models --strict-model-config=true --log-verbose=1", "health": "8000:/v2/health/ready", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "5G", "resource_cpu": "5", "inference_config": "\n---config.pbtxt\n\nname: \"resnet50\"\nplatform: \"pytorch_libtorch\"\nmax_batch_size: 0\ninput \n[\n {\n name: \"INPUT__0\"\n data_type: TYPE_FP32\n format: FORMAT_NCHW\n dims: [ 3, 224, 224 ]\n reshape: {\n shape: [ 1, 3, 224, 224 ]\n }\n }\n]\n \noutput \n[\n {\n name: \"OUTPUT__0\"\n data_type: TYPE_FP32\n dims: [ 1000 ]\n reshape: {\n shape: [ 1, 1000 ]\n }\n }\n]\n \n\nparameters: { key: \"DISABLE_OPTIMIZED_EXECUTION\" value: { string_value:\"true\" } }\nparameters: { key: \"INFERENCE_MODE\" value: { string_value: \"false\" } }\n", "expand": { } }, "onnx-resnet50": { "project_name": "public", "service_name": "resnet50-202208014", "model_name": "resnet50", "service_describe": "onnx 图像分类", "image_name": "ccr.ccs.tencentyun.com/cube-studio/tritonserver:22.07-py3", "model_version": "v2022.08.01.4", "model_path": "https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/resnet50.onnx", "service_type": "triton-server", "env": "", "host": "/v2/models/resnet50", "ports": "8000,8002", "metrics": "", "workdir": "", "command": "wget https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/resnet50.onnx && mkdir -p /models/resnet50/202208014/ && cp /config/* /models/resnet50/ && cp -r resnet50.onnx /models/resnet50/202208014/model.onnx && tritonserver --model-repository=/models --strict-model-config=true --log-verbose=1", "health": "8000:/v2/health/ready", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "5G", "resource_cpu": "5", "inference_config": "---config.pbtxt\n\nname: \"resnet50\"\nplatform: \"onnxruntime_onnx\"\nbackend: \"onnxruntime\"\nmax_batch_size : 0\n\ninput [\n {\n name: \"input_name\"\n data_type: TYPE_FP32\n format: FORMAT_NCHW\n dims: [ 3, 224, 224 ]\n reshape { shape: [ 1, 3, 224, 224 ] }\n }\n]\noutput [\n {\n name: \"output_name\"\n data_type: TYPE_FP32\n dims: [ 1000 ]\n reshape { shape: [ 1, 1000 ] }\n }\n]\n\nparameters { key: \"intra_op_thread_count\" value: { string_value: \"10\" } }\nparameters { key: \"execution_mode\" value: { string_value: \"1\" } }\nparameters { key: \"inter_op_thread_count\" value: { string_value: \"10\" } }\n", "expand": { "help_url": "https://github.com/data-infra/cube-studio/tree/main/images/serving/triton-server/example" } } } --- ## File: myapp/init/init-pipeline.json { "yolov8": { "pipeline": { "name": "yolov8", "describe": "深度学习：目标识别训练部署", "project": "public", "parameter": { "demo": "true", "img": "/static/assets/images/pipeline/yolo.jpg" }, "dag_json": { "download-data": { "upstream": [] }, "yolov8-object-recognition": { "upstream": [ "download-data" ] }, "deploy-yolov8-web-service": { "upstream": [ "yolov8-object-recognition" ] } } }, "tasks": [ { "job_templete": "自定义镜像", "name": "download-data", "label": "下载处理标注数据", "volume_mount": "kubeflow-user-workspace(pvc):/mnt/", "args": { "images": "ccr.ccs.tencentyun.com/cube-studio/ubuntu-gpu:cuda11.8.0-cudnn8-python3.9", "workdir": "/mnt/{{creator}}/", "command": "rm -rf coco.zip && wget https://cube-studio.oss-cn-hangzhou.aliyuncs.com/pipeline/coco.zip --no-check-certificate && unzip -o coco.zip && cd coco_data_sample && bash reset_file.sh" } }, { "job_templete": "yolov8", "name": "yolov8-object-recognition", "label": "目标识别训练", "volume_mount": "kubeflow-user-workspace(pvc):/mnt/", "resource_memory": "5G", "resource_cpu": "5", "args": { "--train": "/mnt/{{creator}}/coco_data_sample/train.txt", "--val": "/mnt/{{creator}}/coco_data_sample/valid.txt", "--classes": "person,bicycle,car,motorcycle,airplane,bus,train,truck,boat,trafficlight,firehydrant,stopsign,parkingmeter,bench,bird,cat,dog,horse,sheep,cow,elephant,bear,zebra,giraffe,backpack,umbrella,handbag,tie,suitcase,frisbee,skis,snowboard,sportsball,kite,baseballbat,baseballglove,skateboard,surfboard,tennisracket,bottle,wineglass,cup,fork,knife,spoon,bowl,banana,apple,sandwich,orange,broccoli,carrot,hotdog,pizza,donut,cake,chair,couch,pottedplant,bed,diningtable,toilet,tv,laptop,mouse,remote,keyboard,cellphone,microwave,oven,toaster,sink,refrigerator,book,clock,vase,scissors,teddybear,hairdrier,toothbrush", "--batch_size": "1", "--epoch": "10", "--weights": "/yolov8/yolov8n.pt", "--save_model_path": "/mnt/{{creator}}/coco_data_sample/yolov8_best.pt" } }, { "job_templete": "deploy-service", "name": "deploy-yolov8-web-service", "volume_mount": "kubeflow-user-workspace(pvc):/mnt/", "label": "部署模型web服务", "args": { "--project_name": "public", "--label": "目标识别推理服务", "--model_name": "yolov8", "--model_version": "v2024.07.01.1", "--model_path": "/mnt/{{creator}}/coco_data_sample/yolov8_best.pt", "--service_type": "serving", "--images": "ccr.ccs.tencentyun.com/cube-studio/yolov8:20250801", "--volume_mount": "kubeflow-user-workspace(pvc):/mnt,/data/k8s/kubeflow/labelstudio(hostpath):/labelstudio/", "--working_dir": "/yolov8", "--command": "python server.py", "--env": "MODELPATH=/mnt/{{creator}}/coco_data_sample/yolov8_best.pt\nYOLO_EXAMPLE=/mnt/{{creator}}/coco_data_sample/images/train2014/COCO_train2014_000000000597.jpg", "--ports": "8080", "--replicas": "1", "--resource_memory": "2G", "--resource_cpu": "1", "--resource_gpu": "0" } } ] }, "ml": { "pipeline": { "name": "ml", "describe": "机器学习：决策树训练部署", "project": "public", "parameter": { "demo": "true", "img": "/static/assets/images/pipeline/tree.jpg" }, "dag_json": { "data-import": {}, "data-process": { "upstream": [ "data-import" ] }, "model-train": { "upstream": [ "data-process" ] }, "model-val": { "upstream": [ "model-train" ] }, "model-register": { "upstream": [ "model-val" ] }, "model-offline-inference": { "upstream": [ "model-val" ] }, "deploy-service": { "upstream": [ "model-register" ] } }, "global_env": "", "expand": [ { "id": "model-train", "type": "dataSet", "position": { "x": 384, "y": 232 }, "data": { "info": { "describe": "decision-tree算法" }, "name": "model-train", "label": "模型训练" } }, { "id": "model-val", "type": "dataSet", "position": { "x": 384, "y": 328 }, "data": { "info": { "describe": "decision-tree算法" }, "name": "model-val", "label": "模型评估" } }, { "id": "model-offline-inference", "type": "dataSet", "position": { "x": 224, "y": 440 }, "data": { "info": { "describe": "decision-tree算法" }, "name": "model-offline-inference", "label": "模型离线推理" } }, { "id": "data-import", "type": "dataSet", "position": { "x": 380, "y": -16 }, "data": { "info": { "describe": "datax异构数据源同步" }, "name": "data-import", "label": "数据导入" } }, { "id": "data-process", "type": "dataSet", "position": { "x": 380, "y": 111 }, "data": { "info": { "describe": "使用用户自定义镜像作为运行镜像" }, "name": "data-process", "label": "数据处理" } }, { "id": "model-register", "type": "dataSet", "position": { "x": 524, "y": 447 }, "data": { "info": { "describe": "注册模型" }, "name": "model-register", "label": "模型注册" } }, { "id": "deploy-service", "type": "dataSet", "position": { "x": 524, "y": 556 }, "data": { "info": { "describe": "模型部署推理服务" }, "name": "deploy-service", "label": "服务部署" } }, { "source": "data-import", "arrowHeadType": "arrow", "target": "data-process", "id": "logic__edge-data-importnull-data-processnull" }, { "source": "data-process", "arrowHeadType": "arrow", "target": "model-train", "id": "logic__edge-data-processnull-model-trainnull" }, { "source": "model-train", "arrowHeadType": "arrow", "target": "model-val", "id": "logic__edge-model-trainnull-model-valnull" }, { "source": "model-val", "arrowHeadType": "arrow", "target": "model-offline-inference", "id": "logic__edge-model-valnull-model-offline-inferencenull" }, { "source": "model-val", "arrowHeadType": "arrow", "target": "model-register", "id": "logic__edge-model-valnull-model-registernull" }, { "source": "model-register", "arrowHeadType": "arrow", "target": "deploy-service", "id": "logic__edge-model-registernull-deploy-servicenull" } ] }, "tasks": [ { "job_templete": "decision-tree", "name": "model-train", "label": "模型训练", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "args": { "--train_dataset": "/mnt/{{creator}}/pipeline/example/ml/train.csv", "--label_columns": "y", "--save_model_dir": "/mnt/{{creator}}/pipeline/example/ml/", "--feature_columns": "age,duration,campaign,pdays,previous,emp_var_rate,cons_price_idx,cons_conf_idx,euribor3m,nr_employed", "--model_params": { "max_depth": 5, "min_samples_leaf": 10 }, "--inference_dataset": "", "--val_dataset": "" } }, { "job_templete": "decision-tree", "name": "model-val", "label": "模型评估", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "args": { "--train_dataset": "", "--label_columns": "y", "--save_model_dir": "/mnt/{{creator}}/pipeline/example/ml/", "--feature_columns": "age,duration,campaign,pdays,previous,emp_var_rate,cons_price_idx,cons_conf_idx,euribor3m,nr_employed", "--model_params": "", "--inference_dataset": "", "--val_dataset": "/mnt/{{creator}}/pipeline/example/ml/train.csv" } }, { "job_templete": "decision-tree", "name": "model-offline-inference", "label": "模型离线推理", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "args": { "--train_dataset": "", "--label_columns": "", "--save_model_dir": "/mnt/{{creator}}/pipeline/example/ml/", "--feature_columns": "age,duration,campaign,pdays,previous,emp_var_rate,cons_price_idx,cons_conf_idx,euribor3m,nr_employed", "--model_params": "", "--inference_dataset": "/mnt/{{creator}}/pipeline/example/ml/inference.csv", "--val_dataset": "" } }, { "job_templete": "datax", "name": "data-import", "label": "数据导入", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "args": { "-f": "/mnt/{{creator}}/pipeline/example/ml/mysql-csv.json" } }, { "job_templete": "自定义镜像", "name": "data-process", "label": "数据处理", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "args": { "images": "ccr.ccs.tencentyun.com/cube-studio/ubuntu-gpu:cuda11.8.0-cudnn8-python3.9", "workdir": "/mnt/{{creator}}/pipeline/example/ml/", "command": "sh fix.sh" } }, { "job_templete": "model-register", "name": "model-register", "label": "模型注册", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "args": { "--project_name": "public", "--model_name": "decision-tree", "--model_version": "{{ datetime.datetime.now().strftime('v%Y.%m.%d.1') }}", "--model_path": "/mnt/{{creator}}/pipeline/example/ml/decisionTree_model.pkl", "--model_metric": "", "--describe": "决策树模型", "--framework": "sklearn", "--inference_framework": "ml-server" } }, { "job_templete": "deploy-service", "name": "deploy-service", "label": "服务部署", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "args": { "--project_name": "public", "--label": "决策树模型", "--model_name": "decisiontree", "--model_version": "v2023.11.01.1", "--model_path": "/mnt/{{creator}}/pipeline/example/ml/decisionTree_model.pkl", "--service_type": "ml-server", "--images": "ccr.ccs.tencentyun.com/cube-studio/ml-server:20240601", "--working_dir": "", "--command": "", "--env": "", "--host": "", "--ports": "80", "--replicas": "1", "--resource_memory": "2G", "--resource_cpu": "2", "--resource_gpu": "0", "--volume_mount": "kubeflow-user-workspace(pvc):/mnt", "--inference_config": "---config.json\n\n[\n {\n \"name\": \"decisiontree\",\n \"model_path\": \"/mnt/admin/pipeline/example/ml/decisionTree_model.pkl\",\n \"framework\": \"sklearn\",\n \"version\": \"v2023.10.01.1\",\n \"output\": [\n { \"name\": \"y\", \"type\": \"enum\", \"choices\": [0,1] }\n ],\n \"input\": [\n { \"name\": \"age\", \"type\": \"int\" },\n { \"name\": \"duration\", \"type\": \"int\" },\n { \"name\": \"campaign\", \"type\": \"int\" },\n { \"name\": \"pdays\", \"type\": \"int\" },\n { \"name\": \"previous\", \"type\": \"enum\", \"choices\": [0,1,2,3,4,5,6] },\n { \"name\": \"emp_var_rate\", \"type\": \"float\" },\n { \"name\": \"cons_price_idx\", \"type\": \"float\" },\n { \"name\": \"cons_conf_idx\", \"type\": \"float\" },\n { \"name\": \"euribor3m\", \"type\": \"float\" },\n { \"name\": \"nr_employed\", \"type\": \"float\" }\n ],\n \"example\": [\n { \"age\": 30, \"duration\": 487, \"campaign\": 2, \"pdays\": 999, \"previous\": 0, \"emp_var_rate\": -1.8, \"cons_price_idx\": 92.893, \"cons_conf_idx\": -46.2, \"euribor3m\": 1.313, \"nr_employed\": 5099.1 }\n ],\n \"enable\": true\n }\n]" } } ] }, "deepseek": { "pipeline": { "name": "deepseek", "describe": "大模型：deepseek微调部署", "project": "public", "parameter": { "demo": "true", "img": "/static/assets/images/pipeline/deepseek.jpg" }, "dag_json": { "task-data-process": {}, "deepseek-lora": { "upstream": [ "task-data-process" ] }, "deploy-deepseek": { "upstream": [ "deepseek-lora" ] } }, "global_env": "", "expand": [ { "id": "deploy-deepseek", "type": "dataSet", "position": { "x": 224, "y": 442 }, "data": { "info": { "describe": "模型部署推理服务" }, "name": "deploy-deepseek", "label": "部署deepseek" } }, { "id": "task-data-process", "type": "dataSet", "position": { "x": 224, "y": 133 }, "data": { "info": { "describe": "使用用户自定义镜像作为运行镜像" }, "name": "task-data-process", "label": "数据准备" } }, { "id": "deepseek-lora", "type": "dataSet", "position": { "x": 224, "y": 272 }, "data": { "info": { "describe": "deepseek lora微调" }, "name": "deepseek-lora", "label": "deepseek lora微调" } }, { "source": "task-data-process", "arrowHeadType": "arrow", "target": "deepseek-lora", "id": "logic__edge-task-null-deepseek-null" }, { "source": "deepseek-lora", "arrowHeadType": "arrow", "target": "deploy-deepseek", "id": "logic__edge-deepseek-null-deploy-deepseeknull" } ] }, "tasks": [ { "job_templete": "deploy-service", "name": "deploy-deepseek", "label": "部署deepseek", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "resource_rdma": "0", "args": { "--project_name": "public", "--label": "deepseek推理部署", "--model_name": "deepseek", "--model_version": "v2025.03.01.1", "--model_path": "/mnt/{{creator}}/pipeline/example/deepseek/result/exported_model", "--service_type": "vllm", "--images": "vllm/vllm-openai:v0.8.5.post1", "--working_dir": "", "--command": "python3 -m vllm.entrypoints.openai.api_server --trust-remote-code --max-model-len 8192 --model $KUBEFLOW_MODEL_PATH --host 0.0.0.0 --port 8000 --dtype float16 --tensor-parallel-size $RESOURCE_GPU --served-model-name $KUBEFLOW_MODEL_NAME", "--env": "HF_ENDPOINT=https://hf-mirror.com", "--host": "/v1/models/", "--ports": "8000", "--replicas": "1", "--resource_memory": "20G", "--resource_cpu": "10", "--resource_gpu": "1", "--volume_mount": "kubeflow-user-workspace(pvc):/mnt", "--inference_config": "", "--metrics": "", "--health": "8000:/v1/models/" } }, { "job_templete": "自定义镜像", "name": "task-data-process", "label": "数据准备", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "resource_rdma": "0", "args": { "images": "ccr.ccs.tencentyun.com/cube-studio/ubuntu-gpu:cuda11.8.0-cudnn8-python3.9", "workdir": "/mnt/{{creator}}/pipeline/example/deepseek/", "command": "python data-process-identiy.py" } }, { "job_templete": "deepseek", "name": "deepseek-lora", "label": "deepseek lora微调", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "40G", "resource_cpu": "10", "resource_gpu": "1", "resource_rdma": "0", "args": { "--model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "--model_path": "/mnt/{{creator}}/pipeline/example/deepseek/DeepSeek-R1-Distill-Qwen-7B/", "--dataset": { "path": "/mnt/{{creator}}/pipeline/example/deepseek/identity.json" }, "--template": "deepseek3", "--finetuning_type": "lora", "--lora_target": "all", "--output_dir": "/mnt/{{creator}}/pipeline/example/deepseek/result/", "--per_device_train_batch_size": "4", "--gradient_accumulation_steps": "4", "--lr_scheduler_type": "cosine", "--logging_steps": "10", "--save_steps": "100", "--learning_rate": "5e-5", "--num_train_epochs": "5", "--max_samples": "500", "--max_grad_norm": "1", "--quantization_bit": "none", "--fp16": "true", "--merge_lora": "true" } } ] }, "electric-bicycle": { "pipeline": { "name": "electric-bicycle", "describe": "示例：电瓶车全流程(自动化标注-微调-部署)", "project": "public", "parameter": {}, "dag_json": { "download-model": {}, "labelstudio-train": { "upstream": [ "download-model" ] }, "yolov8-object-recognition": { "upstream": [ "labelstudio-train" ] }, "deploy-yolov8-web-service": { "upstream": [ "yolov8-object-recognition" ] } }, "global_env": "YYYYMMDD={{datetime.datetime.now().strftime('%Y-%m-%d')}}", "expand": [ { "id": "labelstudio-train", "type": "dataSet", "position": { "x": 400, "y": 247 }, "data": { "info": { "describe": "使用用户自定义镜像作为运行镜像" }, "name": "labelstudio-train", "label": "标注结果转为训练数据" } }, { "id": "yolov8-object-recognition", "type": "dataSet", "position": { "x": 400, "y": 363 }, "data": { "info": { "describe": "yolo目标识别" }, "name": "yolov8-object-recognition", "label": "目标识别训练" } }, { "id": "deploy-yolov8-web-service", "type": "dataSet", "position": { "x": 400, "y": 463 }, "data": { "info": { "describe": "模型部署推理服务" }, "name": "deploy-yolov8-web-service", "label": "部署模型web服务" } }, { "id": "download-model", "type": "dataSet", "position": { "x": 400, "y": 127 }, "data": { "info": { "describe": "使用用户自定义镜像作为运行镜像" }, "name": "download-model", "label": "下载校验数据和预训练模型" } }, { "source": "download-model", "arrowHeadType": "arrow", "target": "labelstudio-train", "id": "logic__edge-download-modelnull-labelstudio-trainnull" }, { "source": "labelstudio-train", "arrowHeadType": "arrow", "target": "yolov8-object-recognition", "id": "logic__edge-labelstudio-trainnull-yolov8-object-recognitionnull" }, { "source": "yolov8-object-recognition", "arrowHeadType": "arrow", "target": "deploy-yolov8-web-service", "id": "logic__edge-yolov8-object-recognitionnull-deploy-yolov8-web-servicenull" } ] }, "tasks": [ { "job_templete": "自定义镜像", "name": "labelstudio-train", "label": "标注结果转为训练数据", "volume_mount": "kubeflow-user-workspace(pvc):/mnt/,/data/k8s/kubeflow/labelstudio(hostpath):/labelstudio/", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "resource_rdma": "0", "args": { "images": "ccr.ccs.tencentyun.com/cube-studio/ubuntu-gpu:cuda11.8.0-cudnn8-python3.9", "workdir": "/mnt/{{creator}}/pipeline/example/yolo/electric-bicycle", "command": "python yolo-data-process.py" } }, { "job_templete": "yolov8", "name": "yolov8-object-recognition", "label": "目标识别训练", "volume_mount": "kubeflow-user-workspace(pvc):/mnt/", "resource_memory": "5G", "resource_cpu": "5", "resource_gpu": "1", "resource_rdma": "0", "args": { "--train": "/mnt/{{creator}}/pipeline/example/yolo/electric-bicycle/dataset/train.txt", "--val": "/mnt/{{creator}}/pipeline/example/yolo/electric-bicycle/val1.txt", "--classes": "Electric-bicycle", "--batch_size": "1", "--epoch": "1", "--weights": "/mnt/{{creator}}/pipeline/example/yolo/electric-bicycle/electric-bicycle-new.pt", "--save_model_path": "/mnt/{{creator}}/pipeline/example/yolo/electric-bicycle/electric-bicycle-new.pt" } }, { "job_templete": "deploy-service", "name": "deploy-yolov8-web-service", "label": "部署模型web服务", "volume_mount": "kubeflow-user-workspace(pvc):/mnt/", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "resource_rdma": "0", "args": { "--project_name": "public", "--label": "目标识别推理服务", "--model_name": "electric-bicycle", "--model_version": "v20240701", "--model_path": "/mnt/{{creator}}/pipeline/example/yolo/electric-bicycle/electric-bicycle-new.pt", "--service_type": "serving", "--images": "ccr.ccs.tencentyun.com/cube-studio/yolov8:2024.07", "--working_dir": "/yolov8", "--command": "python server.py", "--env": "MODELPATH=/mnt/admin/pipeline/example/yolo/electric-bicycle/electric-bicycle-new.pt\nYOLO_SAVE_DIR=/mnt/admin/pipeline/example/yolo/inference/\nYOLO_EXAMPLE=https://cube-studio.oss-cn-hangzhou.aliyuncs.com/labelstudio/vision/electric-bicycle/2020_2582.jpg", "--host": "", "--ports": "8080", "--replicas": "1", "--resource_memory": "5G", "--resource_cpu": "5", "--resource_gpu": "1", "--volume_mount": "kubeflow-user-workspace(pvc):/mnt,/data/k8s/kubeflow/labelstudio(hostpath):/labelstudio/", "--inference_config": "", "--metrics": "", "--health": "" } }, { "job_templete": "自定义镜像", "name": "download-model", "label": "下载校验数据和预训练模型", "volume_mount": "kubeflow-user-workspace(pvc):/mnt", "resource_memory": "2G", "resource_cpu": "2", "resource_gpu": "0", "resource_rdma": "0", "args": { "images": "ccr.ccs.tencentyun.com/cube-studio/ubuntu-gpu:cuda11.8.0-cudnn8-python3.9", "workdir": "/mnt/{{creator}}/pipeline/example/yolo/", "command": "rm -rf electric-bicycle.zip && wget https://cube-studio.oss-cn-hangzhou.aliyuncs.com/pipeline/electric-bicycle/electric-bicycle.zip --no-check-certificate && unzip -n electric-bicycle.zip && cd electric-bicycle && sh reset.sh" } } ] } } --- ## File: myapp/init/init-project.json [ { "type": "org", "name": "推荐中心", "describe": "推荐项目组", "expand": {} }, { "type": "org", "name": "搜索中心", "describe": "搜索项目组", "expand": {} }, { "type": "org", "name": "广告中心", "describe": "广告项目组", "expand": {} }, { "type": "org", "name": "安全中心", "describe": "安全项目组", "expand": {} }, { "type": "org", "name": "多媒体中心", "describe": "多媒体项目组", "expand": {} }, { "type": "job-template", "name": "基础命令", "describe": "python/bash等直接在服务器命令行中执行命令的模板", "expand": {"index": 1} }, { "type": "job-template", "name": "数据导入导出", "describe": "集群与用户机器或其他集群之间的数据迁移", "expand": {"index": 1} }, { "type": "job-template", "name": "数据预处理", "describe": "结构化话数据特征处理", "expand": {"index": 1} }, { "type": "job-template", "name": "数据处理工具", "describe": "数据的单机或分布式处理任务,ray/spark/hadoop/volcanojob", "expand": {"index": 1} }, { "type": "job-template", "name": "基础命令", "describe": "python/bash等直接在服务器命令行中执行命令的模板", "expand": {"index": 1} }, { "type": "job-template", "name": "特征处理", "describe": "特征处理相关", "expand": {"index": 1} }, { "type": "job-template", "name": "机器学习框架", "describe": "传统机器学习框架，sklearn", "expand": {"index": 1} }, { "type": "job-template", "name": "机器学习算法", "describe": "传统机器学习，lr/决策树/gbdt/xgb/fm等", "expand": {"index": 1} }, { "type": "job-template", "name": "深度学习", "describe": "深度框架训练，tf/pytorch/mxnet/mpi/horovod/kaldi等", "expand": {"index": 1} }, { "type": "job-template", "name": "分布式加速", "describe": "tf相关的训练，模型校验，离线预测等功能", "expand": {"index": 1} }, { "type": "job-template", "name": "tf分布式", "describe": "tf相关的训练，模型校验，离线预测等功能", "expand": {"index": 1} } ] --- ## File: myapp/init/init-service.json { "mysql-ui": { "project_name": "public", "service_name": "mysql-ui", "service_describe": "可视化编辑mysql数据库", "image_name": "phpmyadmin:5.2.1", "command": "", "env": "PMA_HOST=mysql-service.infra\nPMA_PORT=3306\nPMA_USER=root\nPMA_PASSWORD=admin", "ports": "80", "expand": { "help_url": "/docs/example/service.md" } }, "redis-ui": { "project_name": "public", "service_name": "redis-ui", "service_describe": "可视化编辑redis数据库", "image_name": "ccr.ccs.tencentyun.com/cube-studio/patrikx3:latest", "command": "", "env": "REDIS_NAME=default\nREDIS_HOST=redis-master.infra\nREDIS_PORT=6379\nREDIS_PASSWORD=admin", "ports": "7843", "expand": { "help_url": "/docs/example/service.md" } }, "mongo-express": { "project_name": "public", "service_name": "mongo-express", "service_describe": "可视化编辑mongo数据库", "image_name": "mongo-express:0.54.0", "command": "", "env": "ME_CONFIG_MONGODB_SERVER=xx.xx.xx.xx\nME_CONFIG_MONGODB_PORT=xx\nME_CONFIG_MONGODB_ENABLE_ADMIN=true\nME_CONFIG_MONGODB_ADMINUSERNAME=xx\nME_CONFIG_MONGODB_ADMINPASSWORD=xx\nME_CONFIG_MONGODB_AUTH_DATABASE=xx\nME_CONFIG_MONGODB_AUTH_USERNAME=xx\nME_CONFIG_MONGODB_AUTH_PASSWORD=xx\nVCAP_APP_HOST=0.0.0.0\nVCAP_APP_PORT=8081\nME_CONFIG_OPTIONS_EDITORTHEME=ambiance", "ports": "8081", "expand": { "help_url": "/docs/example/service.md" } }, "neo4j": { "project_name": "public", "service_name": "neo4j", "service_describe": "可视化编辑图数据库neo4j", "image_name": "ccr.ccs.tencentyun.com/cube-studio/neo4j:4.4", "command": "", "env": "NEO4J_AUTH=neo4j/admin", "ports": "7474,7687", "volume_mount": "kubeflow-user-workspace(pvc):/mnt,/data/k8s/kubeflow/pipeline/workspace/admin/neo4j(hostpath):/var/lib/neo4j/data", "expand": { "help_url": "/docs/example/service.md" } }, "postgresql-ui": { "project_name": "public", "service_name": "postgresql-ui", "service_describe": "可视化编辑postgresql数据库", "image_name": "dpage/pgadmin4", "command": "", "env": "PGADMIN_DEFAULT_EMAIL=admin@tencent.com\nPGADMIN_DEFAULT_PASSWORD=admin", "ports": "80", "expand": { "help_url": "/docs/example/service.md" } }, "es": { "project_name": "public", "service_name": "es", "service_describe": "elasticsearch", "image_name": "elasticsearch:7.12.1", "command": "", "env": "discovery.type=single-node\ncluster.name=es-docker-cluster\nTAKE_FILE_OWNERSHIP=111", "ports": "8080,9200", "expand": { "help_url": "https://github.com/data-infra/cube-studio/wiki" } }, "sd-webui": { "project_name": "public", "service_name": "sd-webui", "service_describe": "sd的webui", "image_name": "ccr.ccs.tencentyun.com/cube-studio/sd-webui:20250301", "command": "python /app/stable-diffusion-webui/webui.py --port 80 --listen --no-half", "env": "", "ports": "80", "resource_gpu": "1", "resource_cpu": "5", "resource_memory": "10G", "volume_mount":"/data/k8s/kubeflow/pipeline/workspace/admin/pipeline/example/sd(hostpath):/app/stable-diffusion-webui/models/Stable-diffusion/", "expand": { "help_url": "https://github.com/data-infra/cube-studio/wiki" } }, "xinference": { "project_name": "public", "service_name": "xinference", "service_describe": "xinference大模型推理", "image_name": "xprobe/xinference:latest", "command": "xinference-local -H 0.0.0.0", "env": "XINFERENCE_HOME=/mnt/{{creator}}/pipeline/example/xinference\nHF_HUB_ENABLE_HF_TRANSFER=1\nHF_ENDPOINT=https://hf-mirror.com\nXINFERENCE_MODEL_SRC=modelscope", "ports": "9997", "resource_gpu": "1", "resource_cpu": "20", "resource_memory": "20G", "expand": { "help_url": "https://github.com/xorbitsai/inference" } } } --- ## File: myapp/init/init-train-model.json { "tf-mnist": { "project_name": "public", "name": "mnist", "version": "v2022.08.01.1", "describe": "tf mnist 图像分类 tfserving推理", "path": "https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/tf-mnist.tar.gz", "framework": "tf", "api_type": "tfserving" }, "pytorch-resnet50": { "project_name": "public", "name": "resnet50", "version": "v2022.08.01.2", "describe": "pytorch resnet50 图像分类 torch-server推理", "path": "https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/resnet50.mar", "framework": "pytorch", "api_type": "torch-server" }, "torchscript-resnet50": { "project_name": "public", "name": "resnet50", "version": "v2022.08.01.3", "describe": "torchscript resnet50 图像分类 triton推理", "path": "https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/resnet50-torchscript.pt", "framework": "pytorch", "api_type": "triton-server" }, "onnx-resnet50": { "project_name": "public", "name": "resnet50", "version": "v2022.08.01.4", "describe": "onnx resnet50 图像分类 triton推理", "path": "https://cube-studio.oss-cn-hangzhou.aliyuncs.com/inference/resnet50.onnx", "framework": "onnx", "api_type": "triton-server" } } --- METRICS --- - Files Extracted: 11 - Estimated Token Budget: ~14661 tokens - Recency Window: Active (< 180 days) - Canonical Reference: https://codewiki.google/github.com/tencentmusic/cube-studio