Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 67 additions & 49 deletions conf/docker/.env.setup
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ MC_COST_OPTIMIZER_ALARM_PORT=9000
MC_COST_OPTIMIZER_ASSET_COLLECTOR_PORT=8091
MC_COST_OPTIMIZER_AZURE_COLLECTOR_PORT=18093
MC_COST_OPTIMIZER_NCP_COLLECTOR_PORT=18094
MC_COST_OPTIMIZER_GCP_COLLECTOR_PORT=28095
MC_COST_OPTIMIZER_AZURE_RIGHTSIZER_PORT=18092
MC_COST_OPTIMIZER_NCP_RIGHTSIZER_PORT=18095
MC_COST_OPTIMIZER_ML_SCORER_PORT=18096
MC_COST_OPTIMIZER_DB_HOST_PORT=3307
MC_APPLICATION_MANAGER_DB_HOST_PORT=18085
MC_APPLICATION_MANAGER_NEXUS_PORT=8081
Expand Down Expand Up @@ -171,66 +173,82 @@ MC_DATA_MANAGER_PUBLIC_HOST=https://${MC_IAM_MANAGER_PUBLIC_DOMAIN}:${MC_DATA_MA
MC_APPLICATION_MANAGER_PROXY_PORT=18084
MC_APPLICATION_MANAGER_PUBLIC_HOST=https://${MC_IAM_MANAGER_PUBLIC_DOMAIN}:${MC_APPLICATION_MANAGER_PROXY_PORT}


# MC-COST-OPTIMIZER
MC_COST_OPTIMIZER_COST_DB_NAME=cost
MC_COST_OPTIMIZER_SLACK_DB_NAME=slack_test
MC_COST_OPTIMIZER_MAIL_DB_NAME=mailing

MC_COST_OPTIMIZER_MYSQL_ROOT_PASSWORD=0000
MC_COST_OPTIMIZER_MYSQL_USER=mcmpcostopti
MC_COST_OPTIMIZER_MYSQL_PASSWORD=0000

MC_COST_OPTIMIZER_COST_DB_URL_NEW=jdbc:mysql://mc-cost-optimizer-db:3306/${MC_COST_OPTIMIZER_COST_DB_NAME}?autoReconnect=true&allowMultiQueries=true&useSSL=false&allowPublicKeyRetrieval=true
MC_COST_OPTIMIZER_COST_DB_URL=jdbc:mariadb://mc-cost-optimizer-db:3306/${MC_COST_OPTIMIZER_COST_DB_NAME}?autoReconnect=true&allowMultiQueries=true&useSSL=false&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&characterEncoding=UTF-8
MC_COST_OPTIMIZER_SLACK_DB_URL=jdbc:mariadb://mc-cost-optimizer-db:3306/${MC_COST_OPTIMIZER_SLACK_DB_NAME}?autoReconnect=true&allowMultiQueries=true&useSSL=false&allowPublicKeyRetrieval=true&characterEncoding=UTF-8
MC_COST_OPTIMIZER_MAIL_DB_URL=jdbc:mariadb://mc-cost-optimizer-db:3306/${MC_COST_OPTIMIZER_MAIL_DB_NAME}?autoReconnect=true&allowMultiQueries=true&useSSL=false&allowPublicKeyRetrieval=true&characterEncoding=UTF-8

MC_COST_OPTIMIZER_COST_ASSET_COLLECTOR_URL=http://mc-cost-optimizer-asset-collector:8091
MC_COST_OPTIMIZER_COST_COLLECTOR_URL=http://mc-cost-optimizer-cost-collector:8081
MC_COST_OPTIMIZER_COST_PROCESSOR_URL=http://mc-cost-optimizer-cost-processor:8082
CO_COST_DB_NAME=cost
CO_SLACK_DB_NAME=slack_test
CO_MAIL_DB_NAME=mailing

CO_MYSQL_ROOT_PASSWORD=0000
CO_MYSQL_USER=mcmpcostopti
CO_MYSQL_PASSWORD=0000
CO_OPENBAO_ENABLED=true
CO_OPENBAO_ADDRESS=http://mc-infra-manager-openbao:${MC_INFRA_MANAGER_OPENBAO_PORT}
CO_OPENBAO_TOKEN=${MC_INFRA_MANAGER_OPENBAO_VAULT_TOKEN}

CO_COST_DB_URL_NEW=jdbc:mysql://mc-cost-optimizer-db:3306/${CO_COST_DB_NAME}?autoReconnect=true&allowMultiQueries=true&useSSL=false&allowPublicKeyRetrieval=true
CO_COST_DB_URL=jdbc:mariadb://mc-cost-optimizer-db:3306/${CO_COST_DB_NAME}?autoReconnect=true&allowMultiQueries=true&useSSL=false&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&characterEncoding=UTF-8
CO_SLACK_DB_URL=jdbc:mariadb://mc-cost-optimizer-db:3306/${CO_SLACK_DB_NAME}?autoReconnect=true&allowMultiQueries=true&useSSL=false&allowPublicKeyRetrieval=true&characterEncoding=UTF-8
CO_MAIL_DB_URL=jdbc:mariadb://mc-cost-optimizer-db:3306/${CO_MAIL_DB_NAME}?autoReconnect=true&allowMultiQueries=true&useSSL=false&allowPublicKeyRetrieval=true&characterEncoding=UTF-8

CO_COST_ASSET_COLLECTOR_URL=http://mc-cost-optimizer-asset-collector:8091
CO_COST_COLLECTOR_URL=http://mc-cost-optimizer-cost-collector:8081
CO_COST_PROCESSOR_URL=http://mc-cost-optimizer-cost-processor:8082

CO_COST_GCP_COLLECTOR_URL=http://mc-cost-optimizer-cost-gcp-collector:8095
CO_GCP_PROJECT_ID=gcp-projectid
CO_GCP_CLIENT_EMAIL=gcp-clientemail
CO_GCP_PRIVATE_KEY_ID=gcp-privatekeyid
CO_GCP_PRIVATE_KEY=gcp-privatekey
CO_GCP_BQ_DATASET=gcp-datasetname
CO_GCP_BQ_TABLE=gcp-bigguerytable
CO_GCP_BATCH_CRON_SCHEDULE=0 0 8 * * ?

## Cost Collector
MC_COST_OPTIMIZER_COST_COLLECT_UNUSED_CRON_SCHEDULE=0 30 0,6 * * ?
MC_COST_OPTIMIZER_COST_COLLECT_CUR_CRON_SCHEDULE=0 0 0,6 * * ?
MC_COST_OPTIMIZER_AWS_CUR_EXPORT_NAME=mcmp-costopti
MC_COST_OPTIMIZER_AWS_CUR_EXPORT_PATH_PREFIX=mcmp-costopti
MC_COST_OPTIMIZER_AWS_ACCESS_KEY_ID=accesskey
MC_COST_OPTIMIZER_AWS_SECRET_ACCESS_KEY=secretkey
CO_COST_COLLECT_UNUSED_CRON_SCHEDULE=0 30 0,6 * * ?
CO_COST_COLLECT_CUR_CRON_SCHEDULE=0 0 0,6 * * ?
CO_AWS_CUR_EXPORT_NAME=mcmp-costopti
CO_AWS_CUR_EXPORT_PATH_PREFIX=mcmp-costopti
CO_AWS_ACCESS_KEY_ID=aws-accesskey
CO_AWS_SECRET_ACCESS_KEY=aws-secretaccesskey

## Cost Processor
MC_COST_OPTIMIZER_COST_PROCESS_UNUSED_CRON_SCHEDULE=0 45 * * * ?
MC_COST_OPTIMIZER_COST_PROCESS_ABNORMAL_CRON_SCHEDULE=0 0 1,7 * * ?
MC_COST_OPTIMIZER_COST_SELECTOR_URL=http://mc-cost-optimizer-cost-selector:8083
MC_COST_OPTIMIZER_ALARM_URL=http://mc-cost-optimizer-alarm-service:9000
CO_COST_PROCESS_UNUSED_CRON_SCHEDULE=0 45 * * * ?
CO_COST_PROCESS_ABNORMAL_CRON_SCHEDULE=0 0 1,7 * * ?
CO_COST_SELECTOR_URL=http://mc-cost-optimizer-cost-selector:8083
CO_ALARM_URL=http://mc-cost-optimizer-alarm-service:9000

## Asset Controller
MC_COST_OPTIMIZER_API_URL=http://mc-cost-optimizer-be:9090
MC_COST_OPTIMIZER_ASSET_MONITORING_SERVER=http://mc-observability-manager:18080
MC_COST_OPTIMIZER_ASSET_COLLECT_BATCH_CRON_SCHEDULE=0 10 * * * ?
CO_API_URL=http://mc-cost-optimizer-be:9090
CO_ASSET_MONITORING_SERVER=http://mc-observability-manager:18080
CO_ASSET_COLLECT_BATCH_CRON_SCHEDULE=0 10 * * * ?

## Cost-Azure-Collector
MC_COST_OPTIMIZER_COST_AZURE_COLLECTOR_URL=http://mc-cost-optimizer-cost-azure-collector:8093
MC_COST_OPTIMIZER_AZURE_TENANT_ID=tenantid
MC_COST_OPTIMIZER_AZURE_CLIENT_ID=clientid
MC_COST_OPTIMIZER_AZURE_CLIENT_SECRET=clientsecret
MC_COST_OPTIMIZER_AZURE_SUBSCRIPTION_ID=subscriptionid
MC_COST_OPTIMIZER_AZURE_SERVICE_BATCH_CRON_SCHEDULE=0 0 0 * * ?
MC_COST_OPTIMIZER_AZURE_VM_BATCH_CRON_SCHEDULE=0 10 0 * * ?
MC_COST_OPTIMIZER_AZURE_VM_RIGHTSIZER_SCHEDULE=0 20 0 * * ?
CO_COST_AZURE_COLLECTOR_URL=http://mc-cost-optimizer-cost-azure-collector:8093
CO_AZURE_TENANT_ID=azure-tenantid
CO_AZURE_CLIENT_ID=azure-clientid
CO_AZURE_CLIENT_SECRET=azure-clientsecret
CO_AZURE_SUBSCRIPTION_ID=azure-subscriptionid
CO_AZURE_SERVICE_BATCH_CRON_SCHEDULE=0 0 0 * * ?
CO_AZURE_VM_BATCH_CRON_SCHEDULE=0 10 0 * * ?
CO_AZURE_VM_RIGHTSIZER_SCHEDULE=0 20 0 * * ?

## Cost-NCP-Collector
MC_COST_OPTIMIZER_COST_NCP_COLLECTOR_URL=http://mc-cost-optimizer-cost-ncp-collector:8094
MC_COST_OPTIMIZER_NCP_IAM_ACCESS_KEY=iamaccesskey
MC_COST_OPTIMIZER_NCP_IAM_SECRET_KEY=iamsecretkey
MC_COST_OPTIMIZER_NCP_SERVICE_BATCH_CRON_SCHEDULE=0 0 0 * * ?
MC_COST_OPTIMIZER_NCP_VM_BATCH_CRON_SCHEDULE=0 10 0 * * ?
MC_COST_OPTIMIZER_NCP_VM_RIGHTSIZER_SCHEDULE=0 20 0 * * ?

## Tumblebug credentials
MC_COST_OPTIMIZER_TUMBLEBUG_USERNAME=spider
MC_COST_OPTIMIZER_TUMBLEBUG_PASSWORD=spider

CO_COST_NCP_COLLECTOR_URL=http://mc-cost-optimizer-cost-ncp-collector:8094
CO_NCP_IAM_ACCESS_KEY=ncp-iamaccesskey
CO_NCP_IAM_SECRET_KEY=ncp-iamsecretkey
CO_NCP_SERVICE_BATCH_CRON_SCHEDULE=0 0 0 * * ?
CO_NCP_VM_BATCH_CRON_SCHEDULE=0 10 0 * * ?
CO_NCP_VM_RIGHTSIZER_SCHEDULE=0 20 0 * * ?

## Cost-ML-Scorer (rightsizing score API, reads cost.asset_compute_metric)
CO_ML_SCORER_URL=http://mc-cost-optimizer-ml-scorer:8096
CO_ML_SCORER_WINDOW_DAYS=30
CO_LLM_KEY_MASTER=

#Tumblebug
CO_TBB_URL=http://mc-infra-manager:1323/tumblebug
CO_TBB_USERNAME=default
CO_TBB_PW=default

# MC-APPLICATION-MANAGER
MC_APPLICATION_MANAGER_POSTGRES_DB=appmgr
Expand Down
139 changes: 60 additions & 79 deletions conf/docker/conf/mc-cost-optimizer/init/01_init_cost_db_ddl.sql
Original file line number Diff line number Diff line change
Expand Up @@ -356,84 +356,6 @@ CREATE TABLE `budget_monthly` (
UNIQUE KEY `uq_csp_year_month_project` (`csp`,`year`,`month`,`project_cd`)
) ENGINE=InnoDB AUTO_INCREMENT=384 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci;

CREATE TABLE TASM_AZR_RSRC_OPT_MODN_L
(
PREV_GENTH VARCHAR(50) NOT NULL COMMENT 'Previous Generation',
NEW_GENTH VARCHAR(50) NULL COMMENT 'Current Generation',
RSRC_TYPE VARCHAR(100) NOT NULL COMMENT 'Resource Type',
RGN_ID VARCHAR(50) NOT NULL COMMENT 'Region',
INST_FAMILY VARCHAR(50) NULL COMMENT 'Instance Family',
PRIMARY KEY (RSRC_TYPE, RGN_ID, PREV_GENTH)
)
CHARSET = utf8mb3;

create index TASM_AZR_RSRC_OPT_MODN_L_IDX_01 ON TASM_AZR_RSRC_OPT_MODN_L (PREV_GENTH);

CREATE TABLE TASM_AZR_VM_PRICING_L
(
SKU VARCHAR(150) NOT NULL,
REGION VARCHAR(50) NOT NULL,
INSTANCE_TYPE VARCHAR(50) NULL,
CORES DOUBLE NULL,
RAM DOUBLE NULL,
SERIES VARCHAR(50) NULL,
OS VARCHAR(50) NULL,
PER_HOUR DOUBLE NULL,
PER_HOUR_ONE_YEAR_RESERVED DOUBLE NULL,
PER_HOUR_THREE_YEAR_RESERVED DOUBLE NULL,
PER_HOUR_SPOT DOUBLE NULL,
IS_VCPU VARCHAR(50) NULL,
AVAILABLE_FOR_ML VARCHAR(50) NULL,
IS_HIDDEN VARCHAR(50) NULL,
DISK_SIZE DOUBLE NULL,
GPU VARCHAR(50) NULL,
PRICING_TYPES VARCHAR(50) NULL,
CREA_DT DATETIME NULL COMMENT '생성일시',
PRIMARY KEY (SKU, REGION)
);

CREATE INDEX IX_TASM_AZR_VM_PRICING_L_02
ON TASM_AZR_VM_PRICING_L (REGION, INSTANCE_TYPE, OS, PRICING_TYPES);

create table TASM_CLOUD_RGN_M
(
CLOUD_VNDR_ID VARCHAR(50) NOT NULL COMMENT 'CLOUD VENDOR ID',
RGN_ID VARCHAR(50) NOT NULL COMMENT 'REGION ID',
RGN_NM VARCHAR(100) NOT NULL COMMENT 'REGION 명',
RGN_CODE VARCHAR(100) NULL,
UNIT_PRICE_RGN VARCHAR(100) NULL,
MT_RGN_NM VARCHAR(100) NULL COMMENT 'REGION 명 (METERING)',
SVC_TYPE VARCHAR(10) NOT NULL COMMENT 'SERVICE TYPE( GLOBAL, CHINA)',
USE_YN VARCHAR(1) DEFAULT 'Y' NOT NULL COMMENT 'CLOUD 사용 여부 (''N'':사용안함, ''Y'':사용)',
CREA_DT DATETIME NOT NULL COMMENT '생성일시',
CREA_ID VARCHAR(50) NOT NULL COMMENT '생성자 ID',
CREA_IPADDR VARCHAR(39) NOT NULL COMMENT '생성자 IP주소',
UPDT_DT DATETIME NULL COMMENT '수정일시',
UPDT_ID VARCHAR(50) NULL COMMENT '수정자 ID',
UPDT_IPADDR VARCHAR(39) NULL COMMENT '수정자 IP주소',
PRIMARY KEY (CLOUD_VNDR_ID, RGN_ID)
)
COMMENT 'Cloud Region Informations' CHARSET = utf8mb3;

create table TASM_NCP_INSTANCE_PRICING_L
(
SKU varchar(150) not null,
REGION varchar(50) not null,
INSTANCE_TYPE varchar(50) null,
SERIES varchar(50) null,
PRODUCT_NAME varchar(150) null,
PRODUCT_CODE varchar(100) null,
CORES double null,
MEMORY double null,
DISK_SIZE double null,
GPU double null,
UNIT_CODE varchar(50) not null,
UNIT_CODE_NAME varchar(150) not null,
USD double null,
CREA_DT datetime null comment '생성일시',
primary key (SKU, REGION, UNIT_CODE)
);

-- Azure/NCP용 Unused Daily Mart 테이블
CREATE TABLE `unused_daily_mart` (
`create_dt` timestamp NOT NULL COMMENT '생성일시 (배치 실행 시점)',
Expand All @@ -448,4 +370,63 @@ CREATE TABLE `unused_daily_mart` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci
COMMENT='Azure/NCP Unused 자원 분석용 일별 메트릭 마트';


-- cost.gcp_billing_raw definition

CREATE TABLE `gcp_billing_raw` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'PK',
`created` datetime DEFAULT NULL COMMENT '수집 시각',
`billing_account_id` varchar(255) DEFAULT NULL COMMENT '청구 계정 ID',
`cost` double DEFAULT NULL COMMENT '발생 비용',
`cost_type` varchar(50) DEFAULT NULL COMMENT '비용 타입 (regular, tax, adjustment, rounding_error)',
`currency` varchar(10) DEFAULT NULL COMMENT '통화 코드 (KRW, USD)',
`currency_conversion_rate` double DEFAULT NULL COMMENT '환율',
`export_time` datetime DEFAULT NULL COMMENT '데이터 추출 시각',
`invoice_month` varchar(6) DEFAULT NULL COMMENT '청구월 (YYYYMM)',
`service_id` varchar(255) DEFAULT NULL COMMENT '서비스 고유 ID',
`service_description` varchar(255) DEFAULT NULL COMMENT '서비스 명칭',
`sku_id` varchar(255) DEFAULT NULL COMMENT 'SKU 고유 ID',
`sku_description` varchar(512) DEFAULT NULL COMMENT 'SKU 상세 명칭',
`project_id` varchar(255) DEFAULT NULL COMMENT '프로젝트 ID',
`project_number` varchar(255) DEFAULT NULL COMMENT '프로젝트 번호',
`project_name` varchar(255) DEFAULT NULL COMMENT '프로젝트 명칭',
`project_ancestry_numbers` varchar(512) DEFAULT NULL COMMENT '상위 조직 경로',
`location` varchar(255) DEFAULT NULL COMMENT '상세 위치',
`location_country` varchar(10) DEFAULT NULL COMMENT '국가 코드',
`location_region` varchar(255) DEFAULT NULL COMMENT '리전',
`location_zone` varchar(255) DEFAULT NULL COMMENT '존',
`usage_start_time` datetime DEFAULT NULL COMMENT '사용 시작 시각',
`usage_end_time` datetime DEFAULT NULL COMMENT '사용 종료 시각',
`usage_amount` double DEFAULT NULL COMMENT '사용량',
`usage_unit` varchar(50) DEFAULT NULL COMMENT '사용량 단위',
`usage_amount_in_pricing_units` double DEFAULT NULL COMMENT '과금 단위 기준 수량',
`usage_pricing_unit` varchar(50) DEFAULT NULL COMMENT '과금 단위',
`adjustment_info_id` varchar(255) DEFAULT NULL COMMENT '조정 ID',
`adjustment_info_description` varchar(512) DEFAULT NULL COMMENT '조정 설명',
`adjustment_info_mode` varchar(50) DEFAULT NULL COMMENT '조정 모드',
`adjustment_info_type` varchar(50) DEFAULT NULL COMMENT '조정 타입',
`labels` text DEFAULT NULL COMMENT '라벨 (JSON)',
`system_labels` text DEFAULT NULL COMMENT '시스템 라벨 (JSON)',
`tags` text DEFAULT NULL COMMENT '태그 (JSON)',
`csp_instanceid` varchar(200) DEFAULT NULL COMMENT 'labels.sys_cspresourceid (servicegroup_meta 조인키)',
`vm_id` varchar(100) DEFAULT NULL COMMENT 'labels.sys_id',
`mci_id` varchar(100) DEFAULT NULL COMMENT 'labels.sys_infraid',
`service_cd` varchar(100) DEFAULT NULL COMMENT 'labels.sys_namespace (ns_id, 매핑 폴백)',
PRIMARY KEY (`id`),
KEY `idx_billing_date` (`billing_account_id`,`invoice_month`),
KEY `idx_project_date` (`project_id`,`usage_start_time`),
KEY `idx_service` (`service_description`,`usage_start_time`),
KEY `idx_gbr_csp_instanceid` (`csp_instanceid`)
) ENGINE=InnoDB AUTO_INCREMENT=579 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci COMMENT='GCP 빌링 원본 데이터';

CREATE TABLE IF NOT EXISTS `provider_keys` (
`id` bigint NOT NULL AUTO_INCREMENT,
`ns_id` varchar(100) NOT NULL COMMENT 'namespace ID',
`provider` varchar(20) NOT NULL COMMENT 'openai | anthropic | google',
`enc_key` text NOT NULL COMMENT 'AES-256-GCM 암호문 (base64)',
`iv` varchar(100) NOT NULL COMMENT 'nonce 12바이트 (base64)',
`tag` varchar(100) NOT NULL COMMENT 'GCM 인증 태그 16바이트 (base64)',
`created_at` timestamp DEFAULT current_timestamp(),
`updated_at` timestamp DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`id`),
UNIQUE KEY `uq_ns_provider` (`ns_id`, `provider`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci COMMENT='LLM 프로바이더 API 키 (AES-256-GCM 암호화 저장)';
Loading