Major refactor

Re-work upload and file processing so they are in the same Python
function. Now I will start exposing other command line options in
the form, like unsafe fixes, excluding fields, etc. Now I see tha
t it is easier to save the POSTed file and process it in the same
function so I don't have to pass around the other POSTed form val
ues as URL query parameters.

Now, as a result of changing the flow above, I also had to make a
change to the way I show the results page. Instead of processing
the file and returning the rendered results to the user directly,
I process the file, save the rendered results to /tmp, and return
a redirect to the user to the results page.

main.py

@@ -29,7 +29,7 @@ def index():
 
 
 @app.route("/", methods=["POST"])
-def upload_file():
+def process():
     uploaded_file = request.files["file"]
     filename = secure_filename(uploaded_file.filename)
 
@@ -43,42 +43,59 @@ def upload_file():
         # generate a base64 representation of the filename to use as a slug
         base64name = b64encode(filename.encode("ascii"))
 
-        return redirect(url_for("process_file", base64slug=base64name))
+        # do we need to use secure_filename again here?
+        input_file = os.path.join(app.config["UPLOAD_PATH"], filename)
+        # write output file with the same name as the input file plus "-cleaned"
+        output_file = os.path.join(
+            app.config["UPLOAD_PATH"], os.path.splitext(filename)[0] + "-cleaned.csv"
+        )
+
+        args = ["-i", input_file, "-o", output_file]
+
+        if "unsafe" in request.form:
+            args.append("-u")
+
+        # run subprocess and capture output as UTF-8 so we get a string instead of
+        # bytes for ansi2html
+        results = subprocess.run(
+            ["csv-metadata-quality"] + args,
+            capture_output=True,
+            encoding="UTF-8",
+        )
+        # convert the output to HTML using ansi2html
+        conv = Ansi2HTMLConverter()
+        stdout_html = conv.convert(results.stdout)
+
+        # render the results to HTML so we can save them for later and allowing
+        # the user to share the results page without posting the file again. We
+        # decode base64name before sending it to convert it from bytes to str.
+        results_html = render_template(
+            "result.html",
+            cli_version=cli_version,
+            filename=filename,
+            stdout=stdout_html,
+            base64name=base64name.decode("ascii"),
+        )
+        # save results to a file so it's easy to have a saved results page when
+        # we don't know the options a user used to POST the form.
+        results_html_file = os.path.join(
+            app.config["UPLOAD_PATH"], base64name.decode("ascii")
+        )
+        with open(results_html_file, "w") as fh:
+            fh.write(results_html)
+
+        return redirect(url_for("results", base64slug=base64name))
 
     return "No file selected"
 
 
 @app.route("/result/<base64slug>")
-def process_file(base64slug):
+def results(base64slug):
-    # get filename from base64-encoded slug
+    results_html_file = os.path.join(app.config["UPLOAD_PATH"], base64slug)
-    filename = b64decode(base64slug).decode("ascii")
+    with open(results_html_file, "r") as fh:
+        results_html = fh.read()
 
-    # do we need to use secure_filename again here?
+    return results_html
-    input_file = os.path.join(app.config["UPLOAD_PATH"], filename)
-    # write output file with the same name as the input file plus "-cleaned"
-    output_file = os.path.join(
-        app.config["UPLOAD_PATH"], os.path.splitext(filename)[0] + "-cleaned.csv"
-    )
-
-    args = ["-i", input_file, "-o", output_file]
-
-    # run subprocess and capture output as UTF-8 so we get a string instead of
-    # bytes for ansi2html
-    results = subprocess.run(
-        ["csv-metadata-quality"] + args,
-        capture_output=True,
-        encoding="UTF-8",
-    )
-    # convert the output to HTML using ansi2html
-    conv = Ansi2HTMLConverter()
-    html = conv.convert(results.stdout)
-    return render_template(
-        "result.html",
-        cli_version=cli_version,
-        filename=filename,
-        stdout=html,
-        base64name=base64slug,
-    )
 
 
 @app.route("/result/<base64slug>/download")

templates/index.html

@@ -5,13 +5,21 @@
     {% include 'header.html' %}
     <main class="flex-shrink-0">
       <div class="container py-3">
-        <p class="lead">The DSpace CSV Metadata Quality Checker is a pipeline of sanity checks and automated fixes for a number of common issues in metadata files.</p>
+        <p class="lead">The DSpace CSV Metadata Quality Checker is a collection of sanity checks and automated fixes for a number of common issues in metadata files.</p>
         <form method="POST" action="" enctype="multipart/form-data">
           <div class="mb-3">
             <label for="formFile" class="form-label">Select a CSV file to process</label>
             <input class="form-control" type="file" id="formFile" name="file" accept=".csv">
           </div>
 
+          <p>Options</p>
+
+          <div class="mb-3 form-check form-switch">
+            <input class="form-check-input" type="checkbox" id="unsafeCheckbox" name="unsafe">
+            <label class="form-check-label" for="unsafeCheckbox" aria-describedby="unsafeHelp">Enable unsafe fixes</label>
+            <div id="unsafeHelp" class="form-text">This will remove newlines and perform <a href="https://withblue.ink/2019/03/11/why-you-need-to-normalize-unicode-strings.html" title='When "Zoë" !== "Zoë". Or why you need to normalize Unicode strings'>normalization of Unicode characters</a>. Read more about these <a href="https://github.com/ilri/csv-metadata-quality#unsafe-fixes">unsafe fixes</a>.</div>
+          </div>
+
           <button type="submit" class="btn btn-primary">Submit</button>
         </form>
       </div>

templates/result.html

@@ -4,19 +4,17 @@
   <body class="d-flex flex-column h-100">
     {% include 'header.html' %}
     <main class="flex-shrink-0">
-      <div class="container">
+      <div class="container py-3">
-        <h1 class="mt-3 fs-2">Results</h1>
+        <p class="lead">The DSpace CSV Metadata Quality Checker is a collection of sanity checks and automated fixes for a number of common issues in metadata files.</p>
-        <p>Processing <code>{{ filename }}</code>. Download <a href="/result/{{ base64name }}/download" title="{{ filename | replace('.csv', '-cleaned.csv') }}">cleaned file</a>.</p>
+        <h3>Results</h3>
-      </div>
+        <p>Results for <code>{{ filename }}</code>. Download <a href="/result/{{ base64name }}/download" title="{{ filename | replace('.csv', '-cleaned.csv') }}">cleaned file</a>.</p>
-    </main>
+        <h3>Log</h3>
-
+        <p>The detailed log of the analysis is below. <span style="color: #00aa00">Green</span> indicates a fix was applied, <span style="color: #aa0000">red</span> indicates an error, and <span style="color: #aa5500">orange</span> indicates a warning.</p>
-    <div class="row justify-content-center">
-      <div class="col-md-10">
         <blockquote>
-        {{ stdout | safe }}
+{{- stdout | safe -}}
         </blockquote>
       </div>
-    </div>
+    </main>
 
     {% include 'footer.html' %}
   </body>