Created
June 15, 2023 15:15
-
-
Save arssher/4195f1d1a089f2589955a32b2615d1e8 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import os | |
| import time | |
| import requests | |
| HEADERS = { | |
| 'Authorization': 'Bearer ' + os.environ['NEON_STAGING_KEY'], | |
| } | |
| for i in range(0, 10000): | |
| start = time.time() | |
| result = requests.post( | |
| 'https://console.stage.neon.tech/api/v2/projects', | |
| headers=HEADERS, | |
| json={ | |
| "project": { | |
| "region_id": "aws-eu-west-1", | |
| } | |
| } | |
| ).json() | |
| project_id = result['project']['id'] | |
| uri = result['connection_uris'][0]['connection_uri'] | |
| operation_id = [op['id'] for op in result['operations'] if op['action'] == 'start_compute'][0] | |
| endpoint_id = result['endpoints'][0]['id'] | |
| print(f"{i}: creating {uri}", end ="", flush=True) | |
| while True: | |
| res = requests.get( | |
| f"https://console.stage.neon.tech/api/v2/projects/{project_id}/operations/{operation_id}", | |
| headers=HEADERS | |
| ) | |
| res.raise_for_status() | |
| op_body = res.json() | |
| if op_body['operation']['status'] == 'finished': | |
| break | |
| print(".", end="", flush=True) | |
| time.sleep(0.5) | |
| stop = time.time() | |
| res = requests.patch( | |
| f"https://console.stage.neon.tech/api/v2/projects/{project_id}/endpoints/{endpoint_id}", | |
| headers=HEADERS, | |
| json = { | |
| "endpoint": { | |
| "suspend_timeout_seconds": 28800 | |
| } | |
| } | |
| ).json() | |
| print("+", end="", flush=True) | |
| print(f" done in {stop - start} seconds") |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import os | |
| import time | |
| import requests | |
| import sys | |
| HEADERS = { | |
| # 'Authorization': 'Bearer ' + os.environ['NEON_STAGING_KEY'], | |
| 'Authorization': 'Bearer ' + 'nghmn2cj534nnlwnajdw50d2jh8tum8leg6cnie1tfeb25ns890ss5cznn77ife6', | |
| } | |
| def get_projects(): | |
| params = {'limit': 100} | |
| all_projects = [] | |
| while True: | |
| projects_res = requests.get( | |
| 'https://console.stage.neon.tech/api/v2/projects', | |
| headers=HEADERS, | |
| params=params | |
| ) | |
| projects_res.raise_for_status() | |
| projects = projects_res.json() | |
| cursor = projects['pagination']['cursor'] | |
| params['cursor'] = cursor | |
| projects = projects['projects'] | |
| print(f"fetched {len(projects)} projects") | |
| if len(projects) == 0: | |
| break | |
| all_projects.extend(projects) | |
| print(f"fetched {len(all_projects)} in total") | |
| return all_projects | |
| projects = get_projects() | |
| # delete in eu-west-1 all but 10k projects | |
| projects = [p for p in projects if p['region_id'] == 'aws-eu-west-1'] | |
| print(f"left {len(projects)} eu-west-1 projects") | |
| projects = projects[10000:] | |
| for i, p in enumerate(projects): | |
| p_id = p['id'] | |
| print(f"deleting {i} project {p_id}") | |
| del_resp = requests.delete( | |
| f"https://console.stage.neon.tech/api/v2/projects/{p_id}", | |
| headers=HEADERS, | |
| ) | |
| del_resp.raise_for_status() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| import os | |
| import time | |
| import requests | |
| import sys | |
| import psycopg2 | |
| import logging | |
| logging.basicConfig( | |
| format='%(asctime)s%(msecs)03d %(levelname)-4s %(message)s', | |
| level=logging.INFO, | |
| datefmt='%Y-%m-%d %H:%M:%S.') | |
| HEADERS = { | |
| 'Authorization': 'Bearer ' + os.environ['NEON_STAGING_KEY'], | |
| } | |
| USER = 'ars' | |
| def get_projects(): | |
| params = {'limit': 100} | |
| all_projects = [] | |
| while True: | |
| projects_res = requests.get( | |
| 'https://console.stage.neon.tech/api/v2/projects', | |
| headers=HEADERS, | |
| params=params | |
| ) | |
| projects_res.raise_for_status() | |
| projects = projects_res.json() | |
| cursor = projects['pagination']['cursor'] | |
| params['cursor'] = cursor | |
| projects = projects['projects'] | |
| logging.info(f"fetched {len(projects)} projects") | |
| if len(projects) == 0: | |
| break | |
| all_projects.extend(projects) | |
| logging.info(f"fetched {len(all_projects)} in total") | |
| return all_projects | |
| # get first endpoint of the project | |
| def get_endpoint(p_id): | |
| eps_res = requests.get( | |
| f"https://console.stage.neon.tech/api/v2/projects/{p_id}/endpoints", | |
| headers=HEADERS, | |
| ) | |
| eps_res.raise_for_status() | |
| ep = eps_res.json()['endpoints'][0] | |
| return ep | |
| # set endpoint size to min 0.25 to save the bill | |
| def set_min_size(project_id, endpoint_id): | |
| logging.info(f"setting min size on {project_id} {endpoint_id}") | |
| res = requests.patch( | |
| f"https://console.stage.neon.tech/api/v2/projects/{project_id}/endpoints/{endpoint_id}", | |
| headers=HEADERS, | |
| json = { | |
| "endpoint": { | |
| "autoscaling_limit_min_cu": 0.25, | |
| "autoscaling_limit_max_cu": 0.25, | |
| } | |
| } | |
| ) | |
| res.raise_for_status() | |
| logging.info("setting min size done") | |
| # logging.info(f"patch res: {res.json()}") | |
| # get connection string of random project endpoint | |
| def project_uri(p_id, i=0): | |
| ep = get_endpoint(p_id) | |
| eps_res = requests.get( | |
| f"https://console.stage.neon.tech/api/v2/projects/{p_id}/endpoints", | |
| headers=HEADERS, | |
| ) | |
| eps_res.raise_for_status() | |
| ep = eps_res.json()['endpoints'][0] | |
| ep_host = ep['host'] | |
| branch_id = ep['branch_id'] | |
| password_res = requests.get( | |
| f"https://console.stage.neon.tech/api/v2/projects/{p_id}/branches/{branch_id}/roles/{USER}/reveal_password", | |
| headers=HEADERS) | |
| password_res.raise_for_status() | |
| password = password_res.json()['password'] | |
| uri = f"postgres://{USER}:{password}@{ep_host}/neondb" | |
| logging.info(f"fetched {i} uri {uri}") | |
| time.sleep(0.1) | |
| return uri | |
| # write something to pg | |
| def do_something(uri): | |
| start = time.time() | |
| pg_conn = psycopg2.connect(dsn=uri, connect_timeout=30) | |
| pg_conn.autocommit = True | |
| cur = pg_conn.cursor() | |
| cur.execute("SET statement_timeout = '5s'") | |
| cur.execute("create table if not exists t(ts timestamptz, haha text)") | |
| cur.execute("insert into t values (now(), 'hoho')") | |
| end = time.time() | |
| logging.info(f"took {end - start}s") | |
| projects = get_projects() | |
| projects = [p for p in projects if p['region_id'] == 'aws-eu-west-1'] | |
| # projects = projects[:3] | |
| # Learn uri and at the same time issue a query to gradually wake up as learning | |
| # uri takes a lot of time. | |
| projects_connstrs = [] | |
| for i, p in enumerate(projects): | |
| project_id = p['id'] | |
| try: | |
| ep = get_endpoint(project_id) | |
| endpoint_id = ep['id'] | |
| # update suspend_timeout | |
| ep_patch_res = requests.patch( | |
| f"https://console.stage.neon.tech/api/v2/projects/{project_id}/endpoints/{endpoint_id}", | |
| headers=HEADERS, | |
| json = { | |
| "endpoint": { | |
| "suspend_timeout_seconds": 93600 | |
| } | |
| } | |
| ) | |
| ep_patch_res.raise_for_status() | |
| set_min_size(project_id, endpoint_id) | |
| uri = project_uri(project_id, i=i) | |
| projects_connstrs.append(uri) | |
| logging.info(f"doing smth with {i} project {uri}") | |
| do_something(uri) | |
| except Exception as e: | |
| logging.info(f"EXCEPTION: {e}") | |
| logging.info(f"fetched connstrings for all projects") | |
| while True: | |
| for i, uri in enumerate(projects_connstrs): | |
| logging.info(f"doing smth with {i} project {uri}") | |
| do_something(uri) | |
| logging.info(f"went over all {i} projects") | |
| time.sleep(100) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment