Last active
June 21, 2021 07:40
-
-
Save phucngta/2a97b90a3862c2e00288b7c04a38380c to your computer and use it in GitHub Desktop.
Handle Post-deployment-jobs in Gitlab
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| import gitlab | |
| import os | |
| import subprocess | |
| import json | |
| import requests | |
| import re | |
| import odoorpc | |
| from datetime import date | |
| # https://github.com/matthewwithanm/python-markdownify | |
| from markdownify import markdownify as md | |
| SERVER_URL = "https://gitlab.besco.vn/" | |
| SLACK_WEBHOOK = os.environ['SLACK_WEBHOOK'] | |
| DEBUG_SLACK = os.getenv('DEBUG_SLACK', "False") | |
| DEBUG_SLACK = json.loads(DEBUG_SLACK.lower()) | |
| GITLAB_PERSONAL_TOKEN = os.environ['GITLAB_PERSONAL_TOKEN'] | |
| PROJECT_ID = os.environ['CI_PROJECT_ID'] | |
| CI_PROJECT_NAME = os.environ['CI_PROJECT_NAME'] | |
| CI_PROJECT_URL = os.environ['CI_PROJECT_URL'] | |
| CI_COMMIT_REF_NAME = os.environ['CI_COMMIT_REF_NAME'] | |
| CI_PIPELINE_ID = os.environ['CI_PIPELINE_ID'] | |
| CI_PIPELINE_URL = os.environ['CI_PIPELINE_URL'] | |
| CI_JOB_STATUS = os.environ['CI_JOB_STATUS'] | |
| CI_JOB_URL = os.environ['CI_JOB_URL'] | |
| CI_JOB_ID = os.environ['CI_JOB_ID'] | |
| BMS_LOGIN_USER = os.environ['BMS_LOGIN_USER'] | |
| BMS_LOGIN_PWD = os.environ['BMS_LOGIN_PWD'] | |
| odoo = odoorpc.ODOO('bms.besco.vn', port=8069) | |
| odoo.login('BMS_MAIN', BMS_LOGIN_USER, BMS_LOGIN_PWD) | |
| TASK_OBJECT = odoo.env['project.task'] | |
| TASKS_INFO = {} | |
| def fetch_task_bms_info(task_numbers): | |
| task_ids = TASK_OBJECT.search_read( | |
| [('task_no', 'in', task_numbers)], ['id']) | |
| for task_id in task_ids: | |
| task_bms_id = task_id.get('id') | |
| if task_bms_id: | |
| task = TASK_OBJECT.browse(task_bms_id) | |
| TASKS_INFO.update({ | |
| task.task_no: { | |
| 'id': task.id, | |
| 'name': task.name, | |
| 'description': md(task.description, strip=['img']), | |
| 'tag_name': task.tag_id.name, | |
| 'milestone_name': task.milestone_id.name, | |
| 'staging_date': task.staging_date, | |
| 'production_date': task.production_date, | |
| 'stage_code': task.stage_code, | |
| 'bms_url': f"https://bms.besco.vn/web#id={task_bms_id}&view_type=form&model=project.task&menu_id=95&action=149" | |
| } | |
| }) | |
| print(" INFO TASKS BMS", TASKS_INFO) | |
| def update_deployment_stage_task_bms(): | |
| if CI_JOB_STATUS == "success" and CI_COMMIT_REF_NAME == 'staging': | |
| for task_no, info in TASKS_INFO.items(): | |
| print(f'task_no {task_no} - staging_date {info.get("staging_date")} - production_date {info.get("production_date")} - stage_code {info.get("stage_code")}') | |
| if info.get("staging_date") and info.get('stage_code') == 'done' and \ | |
| not info.get('production_date'): | |
| task = TASK_OBJECT.browse(info.get("id")) | |
| print(f'Update Production Date {date.today()} for task {task_no}') | |
| task.production_date = date.today() | |
| # if DEBUG_SLACK: | |
| # if CI_JOB_STATUS == "success" and CI_COMMIT_REF_NAME == 'develop': | |
| # for task_no, info in TASKS_INFO.items(): | |
| # print(f'task_no {task_no} - staging_date {info.get("staging_date")} - production_date {info.get("production_date")} - stage_code {info.get("stage_code")}') | |
| # if info.get("staging_date") and info.get("stage_code") == 'done' and \ | |
| # not info.get('production_date') and task_no == 'T15244': | |
| # task = TASK_OBJECT.browse(info.get("id")) | |
| # print(f'Update Production Date {date.today()} for task {task_no}') | |
| # task.production_date = date.today() | |
| def get_latest_deploy_date(): | |
| gl = gitlab.Gitlab(SERVER_URL, private_token=GITLAB_PERSONAL_TOKEN) | |
| project = gl.projects.get(PROJECT_ID) | |
| jobs = project.jobs.list( | |
| query_parameters={'scope': 'success'}, as_list=False) | |
| last_job = project.jobs.get(CI_JOB_ID) | |
| for job in jobs: | |
| if job._attrs['ref'] == CI_COMMIT_REF_NAME \ | |
| and job._attrs['stage'] == 'deploy' \ | |
| and job._attrs['id'] != CI_JOB_ID: | |
| last_job = job | |
| break | |
| latest_deploy_date = last_job._attrs['created_at'] | |
| # create_date = datetime.datetime.strptime(create_date, '%Y-%m-%dT%H:%M:%S.%f%z') | |
| print("Latest Successful Deploy Job", last_job._attrs['id']) | |
| print("Latest Deploy Date", latest_deploy_date) | |
| return latest_deploy_date | |
| def get_need_deployed_merge_requests_str(): | |
| latest_deploy_date = get_latest_deploy_date() | |
| # if DEBUG_SLACK: | |
| # latest_deploy_date = '2021-06-20T07:41:17.191Z' | |
| hash_lst = subprocess.check_output( | |
| ['git', 'log', | |
| '--merges', | |
| f'--after={latest_deploy_date}', | |
| '--first-parent', | |
| f'--pretty=format:%H'], | |
| stderr=subprocess.STDOUT).decode("utf-8").split('\n') | |
| hash_lst = list(filter(None, hash_lst)) | |
| url_hsh_lst = [ | |
| f"<{CI_PROJECT_URL}/commit/{hsh}|:bee:>" for hsh in hash_lst | |
| ] | |
| print("List commit hash:", url_hsh_lst, "\n") | |
| subject_lst = subprocess.check_output( | |
| ['git', 'log', | |
| '--date=format:"%d/%m/%Y %H:%M"', | |
| '--merges', | |
| f'--after={latest_deploy_date}', | |
| '--first-parent', | |
| f'--pretty=format:%s'], | |
| stderr=subprocess.STDOUT | |
| ).decode("utf-8").split('\n') | |
| subject_lst = list(filter(None, subject_lst)) | |
| # =>> Output: ["Merge branch 'T15142' into 'staging'"] | |
| task_lst = [ | |
| sj.split("'")[1] | |
| if len(sj.split("'")) > 1 and sj.split("'")[0] == 'Merge branch ' else sj | |
| for sj in subject_lst | |
| ] | |
| fetch_task_bms_info(task_lst) | |
| task_lst = [ | |
| f"<{TASKS_INFO.get(tsk, {}).get('bms_url', '')}|{tsk}>" | |
| for tsk in task_lst | |
| ] | |
| print("List tasks need deployed:", task_lst, "\n") | |
| body_str = subprocess.check_output( | |
| ['git', 'log', | |
| '--date=format:"%d/%m/%Y %H:%M"', | |
| '--merges', | |
| f'--after={latest_deploy_date}', | |
| '--first-parent', | |
| f'--pretty=format:%bEND'], | |
| stderr=subprocess.STDOUT | |
| ).decode("utf-8") | |
| # =>> Output: '- Test line1\n- Test line2\n\nT15164\n\nSee merge request thanh/besco_minhdang!1750END\n[ADD] channel for debug mode\n\nSee merge request thanh/besco_minhdang!1748END\n[RMV] channel test\n\nSee merge request thanh/besco_minhdang!1747END' | |
| body_lst = re.split('\\n\\nSee merge request.*END', body_str) | |
| body_lst = list(filter(None, body_lst)) | |
| body_lst = [ | |
| # Replace >, _ to quote, and italicized text | |
| body.replace('\n', '_\n>_') | |
| if len(body) > 70 else " => _" + body.replace('\n', ' ') | |
| for body in body_lst | |
| ] | |
| print("Body messages:", body_lst, "\n") | |
| merge_requests_str = "" | |
| if body_lst and task_lst and url_hsh_lst: | |
| mr_with_url_lst = [f"{h_} *`{tsk_}`* {bd_}_" for h_, tsk_, bd_ in | |
| zip(url_hsh_lst, task_lst, body_lst)] | |
| merge_requests_str = "\n".join(mr_with_url_lst) | |
| print("List merge requests need deployed:\n", merge_requests_str, "\n\n") | |
| return merge_requests_str | |
| def send_payload_slack(): | |
| if CI_JOB_STATUS == "failed": | |
| slack_msg_header = f":x: *[{CI_PROJECT_NAME.upper()}]* Deploy to *{CI_COMMIT_REF_NAME.upper()}* failed" | |
| elif CI_JOB_STATUS == "success": | |
| print("DEBUG_MODE", DEBUG_SLACK) | |
| if CI_COMMIT_REF_NAME == "develop" and not DEBUG_SLACK: | |
| print("==> Skip notify successful deployment in integration") | |
| return | |
| slack_msg_header = f":white_check_mark: *[{CI_PROJECT_NAME.upper()}]* Deploy to *{CI_COMMIT_REF_NAME.upper()}* succeeded" | |
| else: | |
| return | |
| print("Slack Msg Header", slack_msg_header) | |
| merge_requests_str = get_need_deployed_merge_requests_str() | |
| if merge_requests_str == "": | |
| print("==> Skip notify: No merge requests were deployed") | |
| return | |
| json_payload = prepare_payload_slack_json( | |
| slack_msg_header, merge_requests_str) | |
| headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} | |
| r = requests.post(SLACK_WEBHOOK, data=json_payload, headers=headers) | |
| print(r.content) | |
| def prepare_payload_slack_json(slack_msg_header, merge_requests_str): | |
| payload = { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": slack_msg_header | |
| } | |
| }, | |
| { | |
| "type": "divider" | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": f"*Job:* <{CI_JOB_URL}|{CI_JOB_ID}>" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": f"*Pipeline:* <{CI_PIPELINE_URL}|{CI_PIPELINE_ID}>" | |
| }, | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": | |
| merge_requests_str | |
| and f"*List merge requests {CI_JOB_STATUS == 'success' and 'were' or 'were _not_'} deployed:* \n" + merge_requests_str | |
| or "*No merge requests were deployed*" | |
| } | |
| } | |
| ] | |
| } | |
| if CI_COMMIT_REF_NAME == "develop": | |
| if CI_JOB_STATUS == "failed": | |
| payload["channel"] = "#mrp-dev" | |
| if DEBUG_SLACK: | |
| payload["channel"] = "#test-slack" | |
| payload_json = json.dumps(payload) | |
| print("Payload Json: ", payload_json, "\n\n") | |
| return payload_json | |
| if __name__ == "__main__": | |
| send_payload_slack() | |
| update_deployment_stage_task_bms() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment