Add step to extract collector name & create col for reconciliation

.gitignore

@@ -0,0 +1,2 @@
+env
+data

Dockerfile

@@ -17,7 +17,8 @@ RUN ls -l /data
 RUN unzip /data/gbif-occs.zip -d /data
 RUN ls -l /data
 COPY ./tab2csv.py /code/tab2csv.py
-RUN python tab2csv.py /data/${GBIF_DOWNLOAD_ID}.csv /data/gbifocc.csv 
+RUN python tab2csv.py /data/${GBIF_DOWNLOAD_ID}.csv /data/gbifocc-temp.csv 
+RUN python extractcollectorname.py /data/gbifocc-temp.csv /data/gbifocc.csv 
 RUN csvs-to-sqlite /data/gbifocc.csv /code/gbifocc.db
 RUN ls -l /code
 RUN sqlite-utils tables /code/gbifocc.db --counts

extractcollectorname.py

@@ -0,0 +1,36 @@
+import argparse
+import pandas as pd
+import bananompy
+from tqdm import tqdm
+tqdm.pandas()
+
+def getFirstFamilyName(s):
+    firstFamilyName = None
+    parsed = bananompy.parse(s)
+    try:
+        firstFamilyName = parsed[0]['parsed'][0]['family']
+    except:
+        pass
+    return firstFamilyName
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("inputfile")
+    parser.add_argument("outputfile")    
+    args = parser.parse_args()
+
+    df = pd.read_csv(args.inputfile, 
+                    encoding='utf8', 
+                    keep_default_na=False, 
+                    na_values=['NONE',''],
+                    on_bad_lines='skip', 
+                    sep=',')
+    # Extract unique recordedBy values
+    df_rb = df[['recordedBy']].drop_duplicates()
+    df_rb['recordedBy_first_familyname'] = df_rb.recordedBy.progress_apply(getFirstFamilyName)
+    # Apply back to main dataframe
+    df = pd.merge(left = df, right=df_rb, left_on='recordedBy', right_on='recordedBy', how='left')
+    # Add column holding collector name and number
+    mask = (df.recordNumber.notnull())
+    df.loc[mask,'collectorNameAndNumber']=df[mask].apply(lambda row: '{} {}'.format(row['recordedBy_first_familyname'],row['recordNumber']),axis=1)
+    df.to_csv(args.outputfile, index=False, sep=',')    

metadata.json

@@ -7,7 +7,7 @@
                     "plugins": {
                         "datasette-reconcile": {
                             "id_field": "gbifID",
-                            "name_field": "recordedBy",
+                            "name_field": "collectorNameAndNumber",
                             "type_field": "basisOfRecord",
                             "type_default": [{
                               "id": "basisOfRecord",

requirements.txt

@@ -3,3 +3,5 @@ datasette-reconcile
 sqlite-utils
 csvs-to-sqlite
 pandas==1.5.3
+bananompy
+tqdm