Clean up code
This commit is contained in:
Louis Elkner-Alfaro
parent
188b16124d
commit
e7877a773c
2 changed files with 102 additions and 37 deletions
|
|
@ -2,23 +2,39 @@ import os
|
|||
import csv
|
||||
import re
|
||||
from dotenv import load_dotenv
|
||||
import requests
|
||||
|
||||
load_dotenv()
|
||||
FOREGEJO_API_TOKEN = os.getenv("FORGEJO_API_TOKEN")
|
||||
FORGEJO_API_TOKEN = os.getenv("FORGEJO_API_TOKEN")
|
||||
|
||||
forgejo_issue_api_string = "/repos/{owner}/{repo}/issues/{index}"
|
||||
forgejo_issue_api_string = "https://git.agaric.com/api/v1/repos/{owner}/{repo}/issues/{index}"
|
||||
|
||||
# {'harvest_project': ('owner', 'repo')}
|
||||
"""
|
||||
projects = {
|
||||
'MASS Continuous Improvement': ('mass', 'mass'),
|
||||
'harvest_project': ('owner', 'repo name')
|
||||
}
|
||||
"""
|
||||
projects = {
|
||||
"MASS Continuous Improvement": ('mass', 'mass'),
|
||||
"Housing Works": ("housingworks", "app-housingworks-net"),
|
||||
}
|
||||
|
||||
def test_parse_notes_section():
|
||||
if (parse_notes_section('Resolved Issue #4, #5 and Issue 6') == ['Issue #4', '#5', 'Issue 6']):
|
||||
print("test parse_notes_section passed")
|
||||
else:
|
||||
print("Test parse_notes_section failed")
|
||||
|
||||
def test_parse_notes_section():
|
||||
if (parse_notes_section('Resolved Issue #4, #5 and Issue 6') == ['4', '5', '6']):
|
||||
print("TEST PARSE NOTES SECTION PASSED")
|
||||
else:
|
||||
print("TEST PARSE NOTES SECTION FAILED")
|
||||
|
||||
def test_get_issue_title_and_url():
|
||||
solution = [
|
||||
"Little slow downs are happening - these often signal that the server memory is getting overwhelmed?",
|
||||
"https://git.agaric.com/housingworks/app-housingworks-net/issues/363"
|
||||
]
|
||||
if get_issue_title_and_url(363) == solution:
|
||||
print("TEST GET ISSUE TITLE AND URL PASSED")
|
||||
else:
|
||||
print("TEST GET ISSUE TITLE AND URL FAILED")
|
||||
|
||||
def parse_notes_section(notes):
|
||||
regex_pattern = r"[Ii]ssue\s*(?:#)?\d+|#\d+"
|
||||
|
|
@ -29,42 +45,72 @@ def parse_notes_section(notes):
|
|||
issue_numbers.append(match)
|
||||
return issue_numbers
|
||||
|
||||
def get_issue_title_and_url(issue_number):
|
||||
global issues_and_urls
|
||||
issues_and_urls = []
|
||||
owner = projects["Housing Works"][0]
|
||||
repo = projects["Housing Works"][1]
|
||||
issue_url = forgejo_issue_api_string.format(owner=owner, repo=repo, index=issue_number)
|
||||
response = requests.get(issue_url, params={"access_token": FORGEJO_API_TOKEN})
|
||||
json_response = response.json()
|
||||
issue_title = json_response['title']
|
||||
issue_url = json_response['html_url']
|
||||
issues_and_urls += [issue_title, issue_url]
|
||||
return [issue_title, issue_url]
|
||||
|
||||
def prompt_for_file(file):
|
||||
file = input("Enter harvest report: ")
|
||||
if not os.path.exists(file):
|
||||
print("THAT FILE DOES NOT EXIST, EXITING PROGRAM")
|
||||
quit()
|
||||
|
||||
print(file)
|
||||
return file
|
||||
|
||||
def parse_harvest_csv(file=None):
|
||||
global issues_and_urls
|
||||
|
||||
if file is None:
|
||||
file = input('Enter harvest report: ')
|
||||
print(file)
|
||||
file = prompt_for_file(file)
|
||||
|
||||
if os.path.exists(file):
|
||||
print('Beginning parsing for issues')
|
||||
print('Beginning parsing for issues')
|
||||
|
||||
with open(file, 'r') as f:
|
||||
csv_reader = csv.DictReader(f)
|
||||
rows = list(csv_reader)
|
||||
original_fieldnames = csv_reader.fieldnames
|
||||
issue_fields = [
|
||||
"First Issue Title",
|
||||
"First Issue URL",
|
||||
"Second Issue Title",
|
||||
"Second Issue URL",
|
||||
"Third Issue Title",
|
||||
"Third Issue URL"
|
||||
]
|
||||
|
||||
issue_fields = [
|
||||
"First Issue Title",
|
||||
"First Issue URL",
|
||||
"Second Issue Title",
|
||||
"Second Issue URL",
|
||||
"Third Issue Title",
|
||||
"Third Issue URL"
|
||||
]
|
||||
|
||||
with open(file, 'r') as f:
|
||||
csv_reader = csv.DictReader(f)
|
||||
rows = list(csv_reader)
|
||||
original_fieldnames = csv_reader.fieldnames
|
||||
modified_fieldnames = original_fieldnames + issue_fields
|
||||
|
||||
with open('modified_csv.csv', 'w', newline='') as csvfile:
|
||||
writer = csv.DictWriter(csvfile, fieldnames = modified_fieldnames)
|
||||
writer.writeheader()
|
||||
row_count = 0
|
||||
for row in rows:
|
||||
issues = parse_notes_section(row['Notes'])
|
||||
issues_dict = dict(zip(issue_fields, issues))
|
||||
row.update(issues_dict)
|
||||
writer.writerow(row)
|
||||
f.close()
|
||||
|
||||
with open('modified_report.csv', 'w', newline='') as csvfile:
|
||||
issues_and_urls = []
|
||||
writer = csv.DictWriter(csvfile, fieldnames = modified_fieldnames)
|
||||
writer.writeheader()
|
||||
row_count = 0
|
||||
for row in rows:
|
||||
issues = parse_notes_section(row['Notes'])
|
||||
for issue in issues:
|
||||
get_issue_title_and_url(issue)
|
||||
issues_dict = dict(zip(issue_fields, issues_and_urls))
|
||||
row.update(issues_dict)
|
||||
issues_and_urls = []
|
||||
writer.writerow(row)
|
||||
|
||||
|
||||
def test_program():
|
||||
parse_harvest_csv("louis_harvest_time_report_from2023-10-01to2023-10-07.csv")
|
||||
def run_program():
|
||||
parse_harvest_csv()
|
||||
|
||||
test_parse_notes_section()
|
||||
test_program()
|
||||
test_get_issue_title_and_url()
|
||||
run_program()
|
||||
|
|
|
|||
19
harvest_report_test.csv
Normal file
19
harvest_report_test.csv
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
"Date","Client","Project","Project Code","Task","Notes","Hours","Hours Rounded","Billable?","Invoiced?","First Name","Last Name","Roles","Employee?","Billable Rate","Billable Amount","Cost Rate","Cost Amount","Currency","External Reference URL"
|
||||
2023-01-11,"MASS Design Group","MASS Continuous Improvement",,"Development","Prepared for and participated in kickoff meeting with Dave, David, and Bob. Issue #363, #362, issue 361",1.5,1.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,225,40,60,"United States Dollar - USD",
|
||||
2023-01-19,"MASS Design Group","MASS Continuous Improvement",,"Planning","Take notes for initial sprint planning meeting for MASS design.",0.75,0.75,"Yes","Yes","Keegan","Rankin",,"Yes",150,112.5,50,37.5,"United States Dollar - USD",
|
||||
2023-01-19,"MASS Design Group","MASS Continuous Improvement",,"Development","Upgraded Drupal and reconciled main branch with upgrade fork.",3.5,3.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,525,40,140,"United States Dollar - USD",
|
||||
2023-01-20,"MASS Design Group","MASS Continuous Improvement",,"Development","Continued upgrading present site, including with spiffy new admin theme.",2.5,2.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,375,40,100,"United States Dollar - USD",
|
||||
2023-02-01,"MASS Design Group","MASS Continuous Improvement",,"Infrastructure","Develop Gitlab-CI routine to auto-deploy test. Configure MFMT environment with site/db/password protection/etc. and various developer improvements.",6,6,"Yes","Yes","Chris","Thompson","Systems Administrator","Yes",150,900,40,240,"United States Dollar - USD",
|
||||
2023-02-01,"MASS Design Group","MASS Continuous Improvement",,"Infrastructure","Helped Chris with final new-site deployment touches.",0.5,0.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,75,40,20,"United States Dollar - USD",
|
||||
2023-02-01,"MASS Design Group","MASS Continuous Improvement",,"Not billed","Scheduled checkin meeting, noted issues share.",0.5,0.5,"Yes","Yes","Benjamin","Melançon",,"Yes",0,0,40,20,"United States Dollar - USD",
|
||||
2023-02-02,"MASS Design Group","MASS Continuous Improvement",,"Troubleshooting","Pair with Chris to troubleshoot the purge queue failing to process.",1.5,1.5,"Yes","Yes","Keegan","Rankin",,"Yes",150,225,50,75,"United States Dollar - USD",
|
||||
2023-02-02,"MASS Design Group","MASS Continuous Improvement",,"Infrastructure","Locate source of issue with purge_queue overgrowth.",3,3,"Yes","Yes","Chris","Thompson","Systems Administrator","Yes",150,450,40,120,"United States Dollar - USD",
|
||||
2023-02-02,"MASS Design Group","MASS Continuous Improvement",,"Development","Identified and helped fix, with Chris, problems with deployment.",1,1,"Yes","Yes","Benjamin","Melançon",,"Yes",150,150,40,40,"United States Dollar - USD",
|
||||
2023-02-03,"MASS Design Group","MASS Continuous Improvement",,"Planning","Attend planning meeting and take notes.",1.5,1.5,"Yes","Yes","Keegan","Rankin",,"Yes",150,225,50,75,"United States Dollar - USD",
|
||||
2023-02-03,"MASS Design Group","MASS Continuous Improvement",,"Development","Met with MASS team with Dave and Keegan and did some followup including on content audit issue and on connecting GitLab to Slack.",2.5,2.5,"Yes","Yes","Benjamin","Melançon",,"Yes",150,375,40,100,"United States Dollar - USD",
|
||||
2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – January Sprint planning",1,1,"Yes","Yes","Agaric","Contractor","contractor","No",150,150,70,70,"United States Dollar - USD",
|
||||
2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – January move to stable9 base theme",0.75,0.75,"Yes","Yes","Agaric","Contractor","contractor","No",150,112.5,70,52.5,"United States Dollar - USD",
|
||||
2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – January MASS: mapping tickets to pages and paragraphs",3.32,3.25,"Yes","Yes","Agaric","Contractor","contractor","No",150,487.5,70,227.5,"United States Dollar - USD",
|
||||
2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – 2/3 Check in",1.5,1.5,"Yes","Yes","Agaric","Contractor","contractor","No",150,225,70,105,"United States Dollar - USD",
|
||||
2023-02-04,"MASS Design Group","MASS Continuous Improvement",,"Design","Dave – January MASS header and mobile menu",4.93,5,"Yes","Yes","Agaric","Contractor","contractor","No",150,750,70,350,"United States Dollar - USD",
|
||||
2023-02-07,"MASS Design Group","MASS Continuous Improvement",,"Infrastructure","Adjust DNS settings for test10 instance (do NOT cache in CloudFlare). Investigate cloning files to test10 site. Too many files (~8GB), so went with adding a fail-over to sites/default/files from live when a file is not found on test10 present.",1.75,1.75,"Yes","Yes","Chris","Thompson","Systems Administrator","Yes",150,262.5,40,70,"United States Dollar - USD",
|
||||
|
Loading…
Reference in a new issue