Major refactor

Re-work upload and file processing so they are in the same Python
function. Now I will start exposing other command line options in
the form, like unsafe fixes, excluding fields, etc. Now I see tha
t it is easier to save the POSTed file and process it in the same
function so I don't have to pass around the other POSTed form val
ues as URL query parameters.

Now, as a result of changing the flow above, I also had to make a
change to the way I show the results page. Instead of processing
the file and returning the rendered results to the user directly,
I process the file, save the rendered results to /tmp, and return
a redirect to the user to the results page.
This commit is contained in:
Alan Orth 2021-03-13 22:11:26 +02:00
parent adc2d06094
commit 198acdb1a7
Signed by: alanorth
GPG Key ID: 0FB860CC9C45B1B9
3 changed files with 65 additions and 42 deletions

79
main.py
View File

@ -29,7 +29,7 @@ def index():
@app.route("/", methods=["POST"]) @app.route("/", methods=["POST"])
def upload_file(): def process():
uploaded_file = request.files["file"] uploaded_file = request.files["file"]
filename = secure_filename(uploaded_file.filename) filename = secure_filename(uploaded_file.filename)
@ -43,42 +43,59 @@ def upload_file():
# generate a base64 representation of the filename to use as a slug # generate a base64 representation of the filename to use as a slug
base64name = b64encode(filename.encode("ascii")) base64name = b64encode(filename.encode("ascii"))
return redirect(url_for("process_file", base64slug=base64name)) # do we need to use secure_filename again here?
input_file = os.path.join(app.config["UPLOAD_PATH"], filename)
# write output file with the same name as the input file plus "-cleaned"
output_file = os.path.join(
app.config["UPLOAD_PATH"], os.path.splitext(filename)[0] + "-cleaned.csv"
)
args = ["-i", input_file, "-o", output_file]
if "unsafe" in request.form:
args.append("-u")
# run subprocess and capture output as UTF-8 so we get a string instead of
# bytes for ansi2html
results = subprocess.run(
["csv-metadata-quality"] + args,
capture_output=True,
encoding="UTF-8",
)
# convert the output to HTML using ansi2html
conv = Ansi2HTMLConverter()
stdout_html = conv.convert(results.stdout)
# render the results to HTML so we can save them for later and allowing
# the user to share the results page without posting the file again. We
# decode base64name before sending it to convert it from bytes to str.
results_html = render_template(
"result.html",
cli_version=cli_version,
filename=filename,
stdout=stdout_html,
base64name=base64name.decode("ascii"),
)
# save results to a file so it's easy to have a saved results page when
# we don't know the options a user used to POST the form.
results_html_file = os.path.join(
app.config["UPLOAD_PATH"], base64name.decode("ascii")
)
with open(results_html_file, "w") as fh:
fh.write(results_html)
return redirect(url_for("results", base64slug=base64name))
return "No file selected" return "No file selected"
@app.route("/result/<base64slug>") @app.route("/result/<base64slug>")
def process_file(base64slug): def results(base64slug):
# get filename from base64-encoded slug results_html_file = os.path.join(app.config["UPLOAD_PATH"], base64slug)
filename = b64decode(base64slug).decode("ascii") with open(results_html_file, "r") as fh:
results_html = fh.read()
# do we need to use secure_filename again here? return results_html
input_file = os.path.join(app.config["UPLOAD_PATH"], filename)
# write output file with the same name as the input file plus "-cleaned"
output_file = os.path.join(
app.config["UPLOAD_PATH"], os.path.splitext(filename)[0] + "-cleaned.csv"
)
args = ["-i", input_file, "-o", output_file]
# run subprocess and capture output as UTF-8 so we get a string instead of
# bytes for ansi2html
results = subprocess.run(
["csv-metadata-quality"] + args,
capture_output=True,
encoding="UTF-8",
)
# convert the output to HTML using ansi2html
conv = Ansi2HTMLConverter()
html = conv.convert(results.stdout)
return render_template(
"result.html",
cli_version=cli_version,
filename=filename,
stdout=html,
base64name=base64slug,
)
@app.route("/result/<base64slug>/download") @app.route("/result/<base64slug>/download")

View File

@ -5,13 +5,21 @@
{% include 'header.html' %} {% include 'header.html' %}
<main class="flex-shrink-0"> <main class="flex-shrink-0">
<div class="container py-3"> <div class="container py-3">
<p class="lead">The DSpace CSV Metadata Quality Checker is a pipeline of sanity checks and automated fixes for a number of common issues in metadata files.</p> <p class="lead">The DSpace CSV Metadata Quality Checker is a collection of sanity checks and automated fixes for a number of common issues in metadata files.</p>
<form method="POST" action="" enctype="multipart/form-data"> <form method="POST" action="" enctype="multipart/form-data">
<div class="mb-3"> <div class="mb-3">
<label for="formFile" class="form-label">Select a CSV file to process</label> <label for="formFile" class="form-label">Select a CSV file to process</label>
<input class="form-control" type="file" id="formFile" name="file" accept=".csv"> <input class="form-control" type="file" id="formFile" name="file" accept=".csv">
</div> </div>
<p>Options</p>
<div class="mb-3 form-check form-switch">
<input class="form-check-input" type="checkbox" id="unsafeCheckbox" name="unsafe">
<label class="form-check-label" for="unsafeCheckbox" aria-describedby="unsafeHelp">Enable unsafe fixes</label>
<div id="unsafeHelp" class="form-text">This will remove newlines and perform <a href="https://withblue.ink/2019/03/11/why-you-need-to-normalize-unicode-strings.html" title='When "Zoë" !== "Zoë". Or why you need to normalize Unicode strings'>normalization of Unicode characters</a>. Read more about these <a href="https://github.com/ilri/csv-metadata-quality#unsafe-fixes">unsafe fixes</a>.</div>
</div>
<button type="submit" class="btn btn-primary">Submit</button> <button type="submit" class="btn btn-primary">Submit</button>
</form> </form>
</div> </div>

View File

@ -4,19 +4,17 @@
<body class="d-flex flex-column h-100"> <body class="d-flex flex-column h-100">
{% include 'header.html' %} {% include 'header.html' %}
<main class="flex-shrink-0"> <main class="flex-shrink-0">
<div class="container"> <div class="container py-3">
<h1 class="mt-3 fs-2">Results</h1> <p class="lead">The DSpace CSV Metadata Quality Checker is a collection of sanity checks and automated fixes for a number of common issues in metadata files.</p>
<p>Processing <code>{{ filename }}</code>. Download <a href="/result/{{ base64name }}/download" title="{{ filename | replace('.csv', '-cleaned.csv') }}">cleaned file</a>.</p> <h3>Results</h3>
</div> <p>Results for <code>{{ filename }}</code>. Download <a href="/result/{{ base64name }}/download" title="{{ filename | replace('.csv', '-cleaned.csv') }}">cleaned file</a>.</p>
</main> <h3>Log</h3>
<p>The detailed log of the analysis is below. <span style="color: #00aa00">Green</span> indicates a fix was applied, <span style="color: #aa0000">red</span> indicates an error, and <span style="color: #aa5500">orange</span> indicates a warning.</p>
<div class="row justify-content-center">
<div class="col-md-10">
<blockquote> <blockquote>
{{ stdout | safe }} {{- stdout | safe -}}
</blockquote> </blockquote>
</div> </div>
</div> </main>
{% include 'footer.html' %} {% include 'footer.html' %}
</body> </body>