MOved create cols into format conv script

Dockerfile

@@ -17,9 +17,9 @@ RUN ls -l /data
 RUN unzip /data/gbif-occs.zip -d /data
 RUN ls -l /data
 COPY ./tab2csv.py /code/tab2csv.py
-COPY ./extractcollectorname.py code/extractcollectorname.py
-RUN python tab2csv.py /data/${GBIF_DOWNLOAD_ID}.csv /data/gbifocc-temp.csv 
-RUN python extractcollectorname.py /data/gbifocc-temp.csv /data/gbifocc.csv 
+#COPY ./extractcollectorname.py /code/extractcollectorname.py
+RUN python tab2csv.py --createcols /data/${GBIF_DOWNLOAD_ID}.csv /data/gbifocc-temp.csv 
+#RUN python extractcollectorname.py /data/gbifocc-temp.csv /data/gbifocc.csv 
 RUN csvs-to-sqlite /data/gbifocc.csv /code/gbifocc.db
 RUN ls -l /code
 RUN sqlite-utils tables /code/gbifocc.db --counts

tab2csv.py

@@ -1,9 +1,13 @@
 import argparse
 import pandas as pd
+import bananompy
+from tqdm import tqdm
+tqdm.pandas()
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument("inputfile")
+    parser.add_argument("-createcols", action='store_true')
     parser.add_argument("outputfile")    
     args = parser.parse_args()
 
@@ -12,4 +16,13 @@ if __name__ == '__main__':
                     keep_default_na=False, 
                     on_bad_lines='skip', 
                     sep='\t')
+    if args.createcols:
+        # Extract unique recordedBy values
+        df_rb = df[['recordedBy']].drop_duplicates()
+        df_rb['recordedBy_first_familyname'] = df_rb.recordedBy.progress_apply(getFirstFamilyName)
+        # Apply back to main dataframe
+        df = pd.merge(left = df, right=df_rb, left_on='recordedBy', right_on='recordedBy', how='left')
+        # Add column holding collector name and number
+        mask = (df.recordNumber.notnull())
+        df.loc[mask,'collectorNameAndNumber']=df[mask].apply(lambda row: '{} {}'.format(row['recordedBy_first_familyname'],row['recordNumber']),axis=1)
     df.to_csv(args.outputfile, index=False, sep=',')