From e7877a773c7ec11dc2a8dc7f838092cd36d3bdd5 Mon Sep 17 00:00:00 2001 From: Louis Elkner-Alfaro Date: Tue, 3 Oct 2023 15:16:42 -0700 Subject: [PATCH] Clean up code --- generate_issues.py | 120 +++++++++++++++++++++++++++------------- harvest_report_test.csv | 19 +++++++ 2 files changed, 102 insertions(+), 37 deletions(-) create mode 100644 harvest_report_test.csv diff --git a/generate_issues.py b/generate_issues.py index cd9cd5d..67dbc35 100644 --- a/generate_issues.py +++ b/generate_issues.py @@ -2,23 +2,39 @@ import os import csv import re from dotenv import load_dotenv +import requests load_dotenv() -FOREGEJO_API_TOKEN = os.getenv("FORGEJO_API_TOKEN") +FORGEJO_API_TOKEN = os.getenv("FORGEJO_API_TOKEN") -forgejo_issue_api_string = "/repos/{owner}/{repo}/issues/{index}" +forgejo_issue_api_string = "https://git.agaric.com/api/v1/repos/{owner}/{repo}/issues/{index}" -# {'harvest_project': ('owner', 'repo')} +""" projects = { - 'MASS Continuous Improvement': ('mass', 'mass'), + 'harvest_project': ('owner', 'repo name') +} +""" +projects = { + "MASS Continuous Improvement": ('mass', 'mass'), + "Housing Works": ("housingworks", "app-housingworks-net"), } -def test_parse_notes_section(): - if (parse_notes_section('Resolved Issue #4, #5 and Issue 6') == ['Issue #4', '#5', 'Issue 6']): - print("test parse_notes_section passed") - else: - print("Test parse_notes_section failed") +def test_parse_notes_section(): + if (parse_notes_section('Resolved Issue #4, #5 and Issue 6') == ['4', '5', '6']): + print("TEST PARSE NOTES SECTION PASSED") + else: + print("TEST PARSE NOTES SECTION FAILED") + +def test_get_issue_title_and_url(): + solution = [ + "Little slow downs are happening - these often signal that the server memory is getting overwhelmed?", + "https://git.agaric.com/housingworks/app-housingworks-net/issues/363" + ] + if get_issue_title_and_url(363) == solution: + print("TEST GET ISSUE TITLE AND URL PASSED") + else: + print("TEST GET ISSUE TITLE AND URL FAILED") def parse_notes_section(notes): regex_pattern = r"[Ii]ssue\s*(?:#)?\d+|#\d+" @@ -29,42 +45,72 @@ def parse_notes_section(notes): issue_numbers.append(match) return issue_numbers +def get_issue_title_and_url(issue_number): + global issues_and_urls + issues_and_urls = [] + owner = projects["Housing Works"][0] + repo = projects["Housing Works"][1] + issue_url = forgejo_issue_api_string.format(owner=owner, repo=repo, index=issue_number) + response = requests.get(issue_url, params={"access_token": FORGEJO_API_TOKEN}) + json_response = response.json() + issue_title = json_response['title'] + issue_url = json_response['html_url'] + issues_and_urls += [issue_title, issue_url] + return [issue_title, issue_url] + +def prompt_for_file(file): + file = input("Enter harvest report: ") + if not os.path.exists(file): + print("THAT FILE DOES NOT EXIST, EXITING PROGRAM") + quit() + + print(file) + return file + def parse_harvest_csv(file=None): + global issues_and_urls + if file is None: - file = input('Enter harvest report: ') - print(file) + file = prompt_for_file(file) - if os.path.exists(file): - print('Beginning parsing for issues') + print('Beginning parsing for issues') - with open(file, 'r') as f: - csv_reader = csv.DictReader(f) - rows = list(csv_reader) - original_fieldnames = csv_reader.fieldnames + issue_fields = [ + "First Issue Title", + "First Issue URL", + "Second Issue Title", + "Second Issue URL", + "Third Issue Title", + "Third Issue URL" + ] - issue_fields = [ - "First Issue Title", - "First Issue URL", - "Second Issue Title", - "Second Issue URL", - "Third Issue Title", - "Third Issue URL" - ] + + with open(file, 'r') as f: + csv_reader = csv.DictReader(f) + rows = list(csv_reader) + original_fieldnames = csv_reader.fieldnames modified_fieldnames = original_fieldnames + issue_fields - with open('modified_csv.csv', 'w', newline='') as csvfile: - writer = csv.DictWriter(csvfile, fieldnames = modified_fieldnames) - writer.writeheader() - row_count = 0 - for row in rows: - issues = parse_notes_section(row['Notes']) - issues_dict = dict(zip(issue_fields, issues)) - row.update(issues_dict) - writer.writerow(row) + f.close() + + with open('modified_report.csv', 'w', newline='') as csvfile: + issues_and_urls = [] + writer = csv.DictWriter(csvfile, fieldnames = modified_fieldnames) + writer.writeheader() + row_count = 0 + for row in rows: + issues = parse_notes_section(row['Notes']) + for issue in issues: + get_issue_title_and_url(issue) + issues_dict = dict(zip(issue_fields, issues_and_urls)) + row.update(issues_dict) + issues_and_urls = [] + writer.writerow(row) -def test_program(): - parse_harvest_csv("louis_harvest_time_report_from2023-10-01to2023-10-07.csv") +def run_program(): + parse_harvest_csv() test_parse_notes_section() -test_program() +test_get_issue_title_and_url() +run_program() diff --git a/harvest_report_test.csv b/harvest_report_test.csv new file mode 100644 index 0000000..452430b --- /dev/null +++ b/harvest_report_test.csv @@ -0,0 +1,19 @@ +"Date","Client","Project","Project Code","Task","Notes","Hours","Hours Rounded","Billable?","Invoiced?","First Name","Last Name","Roles","Employee?","Billable Rate","Billable Amount","Cost Rate","Cost Amount","Currency","External Reference URL" +2023-01-11,"MASS Design Group","MASS Continuous Improvement",,"Development","Prepared for and participated in kickoff meeting with Dave, David, and Bob. Issue #363, #362, issue 361",1.5,1.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,225,40,60,"United States Dollar - USD", +2023-01-19,"MASS Design Group","MASS Continuous Improvement",,"Planning","Take notes for initial sprint planning meeting for MASS design.",0.75,0.75,"Yes","Yes","Keegan","Rankin",,"Yes",150,112.5,50,37.5,"United States Dollar - USD", +2023-01-19,"MASS Design Group","MASS Continuous Improvement",,"Development","Upgraded Drupal and reconciled main branch with upgrade fork.",3.5,3.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,525,40,140,"United States Dollar - USD", +2023-01-20,"MASS Design Group","MASS Continuous Improvement",,"Development","Continued upgrading present site, including with spiffy new admin theme.",2.5,2.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,375,40,100,"United States Dollar - USD", +2023-02-01,"MASS Design Group","MASS Continuous Improvement",,"Infrastructure","Develop Gitlab-CI routine to auto-deploy test. Configure MFMT environment with site/db/password protection/etc. and various developer improvements.",6,6,"Yes","Yes","Chris","Thompson","Systems Administrator","Yes",150,900,40,240,"United States Dollar - USD", +2023-02-01,"MASS Design Group","MASS Continuous Improvement",,"Infrastructure","Helped Chris with final new-site deployment touches.",0.5,0.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,75,40,20,"United States Dollar - USD", +2023-02-01,"MASS Design Group","MASS Continuous Improvement",,"Not billed","Scheduled checkin meeting, noted issues share.",0.5,0.5,"Yes","Yes","Benjamin","Melançon",,"Yes",0,0,40,20,"United States Dollar - USD", +2023-02-02,"MASS Design Group","MASS Continuous Improvement",,"Troubleshooting","Pair with Chris to troubleshoot the purge queue failing to process.",1.5,1.5,"Yes","Yes","Keegan","Rankin",,"Yes",150,225,50,75,"United States Dollar - USD", +2023-02-02,"MASS Design Group","MASS Continuous Improvement",,"Infrastructure","Locate source of issue with purge_queue overgrowth.",3,3,"Yes","Yes","Chris","Thompson","Systems Administrator","Yes",150,450,40,120,"United States Dollar - USD", +2023-02-02,"MASS Design Group","MASS Continuous Improvement",,"Development","Identified and helped fix, with Chris, problems with deployment.",1,1,"Yes","Yes","Benjamin","Melançon",,"Yes",150,150,40,40,"United States Dollar - USD", +2023-02-03,"MASS Design Group","MASS Continuous Improvement",,"Planning","Attend planning meeting and take notes.",1.5,1.5,"Yes","Yes","Keegan","Rankin",,"Yes",150,225,50,75,"United States Dollar - USD", +2023-02-03,"MASS Design Group","MASS Continuous Improvement",,"Development","Met with MASS team with Dave and Keegan and did some followup including on content audit issue and on connecting GitLab to Slack.",2.5,2.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,375,40,100,"United States Dollar - USD", +2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – January Sprint planning",1,1,"Yes","Yes","Agaric","Contractor","contractor","No",150,150,70,70,"United States Dollar - USD", +2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – January move to stable9 base theme",0.75,0.75,"Yes","Yes","Agaric","Contractor","contractor","No",150,112.5,70,52.5,"United States Dollar - USD", +2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – January MASS: mapping tickets to pages and paragraphs",3.32,3.25,"Yes","Yes","Agaric","Contractor","contractor","No",150,487.5,70,227.5,"United States Dollar - USD", +2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – 2/3 Check in",1.5,1.5,"Yes","Yes","Agaric","Contractor","contractor","No",150,225,70,105,"United States Dollar - USD", +2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – January MASS header and mobile menu",4.93,5,"Yes","Yes","Agaric","Contractor","contractor","No",150,750,70,350,"United States Dollar - USD", +2023-02-07,"MASS Design Group","MASS Continuous Improvement",,"Infrastructure","Adjust DNS settings for test10 instance (do NOT cache in CloudFlare). Investigate cloning files to test10 site. Too many files (~8GB), so went with adding a fail-over to sites/default/files from live when a file is not found on test10 present.",1.75,1.75,"Yes","Yes","Chris","Thompson","Systems Administrator","Yes",150,262.5,40,70,"United States Dollar - USD",