francois/HowBadIsMyBatch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding tests test_createDataFrameWithFirstDose() and test_createDataFrameWithSecondDose()

Browse Source
This commit is contained in:
frankknoll
2022-01-29 02:39:42 +01:00
parent 8a496bc4d9
commit c558c90513
1 changed files with 130 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
HowBadIsMyBatch.ipynb
View File

@@ -21,12 +21,18 @@
"metadata": {},
"outputs": [],
"source": [
"def _createDataFrame(vaersDescrs, manufacturer):\n",
" def vaersDescr2DataFrame(vaersDescr):\n",
" return pd.merge(vaersDescr['VAERSDATA'], vaersDescr['VAERSVAX'], left_index = True, right_index = True)\n",
"def _createDataFrame(vaersDescrs, manufacturer, dose):\n",
" def filter(df):\n",
" return df[(df[\"VAX_TYPE\"] == \"COVID19\") & (df[\"VAX_MANU\"] == manufacturer) & (df[\"VAX_DOSE_SERIES\"].str.contains(dose))]\n",
"\n",
" df = pd.concat(map(vaersDescr2DataFrame, vaersDescrs))\n",
" return df[(df[\"VAX_TYPE\"] == \"COVID19\") & (df[\"VAX_MANU\"] == manufacturer)]"
" def vaersDescr2DataFrame(vaersDescr):\n",
" return pd.merge(\n",
" vaersDescr['VAERSDATA'],\n",
" filter(vaersDescr['VAERSVAX']),\n",
" left_index = True,\n",
" right_index = True)\n",
"\n",
" return pd.concat(map(vaersDescr2DataFrame, vaersDescrs))\n"
]
},
{
@@ -36,20 +42,21 @@
"metadata": {},
"outputs": [],
"source": [
"def createDataFrame(dataDir, manufacturer):\n",
"def createDataFrame(dataDir, manufacturer, dose):\n",
" def readVaersDescr(year):\n",
" def read_csv(file, usecols):\n",
" return pd.read_csv(file, index_col = 'VAERS_ID', encoding = 'latin1', low_memory = False, usecols = usecols)\n",
" def read_csv(file, usecols, dtype = {}):\n",
" return pd.read_csv(file, index_col = 'VAERS_ID', encoding = 'latin1', low_memory = False, usecols = usecols, dtype = dtype)\n",
"\n",
" folder = dataDir + \"/\" + year + \"VAERSData/\"\n",
" return {\n",
" 'VAERSDATA': read_csv(folder + year + \"VAERSDATA.csv\", ['VAERS_ID', 'DIED', 'L_THREAT', 'DISABLE']),\n",
" 'VAERSVAX': read_csv(folder + year + \"VAERSVAX.csv\", ['VAERS_ID', 'VAX_DOSE_SERIES', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'])\n",
" 'VAERSVAX': read_csv(folder + year + \"VAERSVAX.csv\", ['VAERS_ID', 'VAX_DOSE_SERIES', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'], dtype = {\"VAX_DOSE_SERIES\": \"string\"})\n",
" }\n",
"\n",
" return _createDataFrame(\n",
" [readVaersDescr(\"2021\"), readVaersDescr(\"2022\")],\n",
" manufacturer)"
" manufacturer,\n",
" dose)"
]
},
{
@@ -97,49 +104,103 @@
" vaersDescrs = [\n",
" {\n",
" 'VAERSDATA': self.createDataFrame(\n",
" [ 'DIED', 'L_THREAT', 'DISABLE'],\n",
" {\n",
" '0916600': ['Y', np.NaN, np.NaN],\n",
" '0916601': [np.NaN, np.NaN, 'Y']\n",
" }),\n",
" index = [\"0916600\", \"0916601\"],\n",
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
" data = [ ['Y', np.NaN, np.NaN],\n",
" [np.NaN, np.NaN, 'Y']]),\n",
" 'VAERSVAX': self.createDataFrame(\n",
" [ 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n",
" {\n",
" '0916600': ['COVID19', 'MODERNA', '037K20A'],\n",
" '0916601': ['COVID19', 'MODERNA', '025L20A']\n",
" })\n",
" index = [\"0916600\", \"0916601\"],\n",
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
" },\n",
" {\n",
" 'VAERSDATA': self.createDataFrame(\n",
" [ 'DIED', 'L_THREAT', 'DISABLE'],\n",
" {\n",
" '1996873': [np.NaN, np.NaN, np.NaN],\n",
" '1996874': [np.NaN, np.NaN, 'Y']\n",
" }),\n",
" index = [\"1996873\", \"1996874\"],\n",
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
" data = [ [np.NaN, np.NaN, np.NaN],\n",
" [np.NaN, np.NaN, 'Y']]),\n",
" 'VAERSVAX': self.createDataFrame(\n",
" [ 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n",
" {\n",
" '1996873': ['HPV9', 'MERCK & CO. INC.', 'R017624'],\n",
" '1996874': ['COVID19', 'MODERNA', '025L20A']\n",
" })\n",
" index = [\"1996873\", \"1996874\"],\n",
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
" data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n",
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
" }\n",
" ]\n",
" \n",
" # When\n",
" dataFrame = _createDataFrame(vaersDescrs, \"MODERNA\")\n",
" dataFrame = _createDataFrame(vaersDescrs, \"MODERNA\", '1')\n",
" \n",
" # Then\n",
" dataFrameExpected = self.createDataFrame(\n",
" [ 'DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n",
" {\n",
" '0916600': ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '037K20A'],\n",
" '0916601': [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A'],\n",
" '1996874': [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A']\n",
" })\n",
" index = [\"0916600\", \"0916601\", \"1996874\"],\n",
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
" data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '037K20A', '1'],\n",
" [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A', '1'],\n",
" [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A', '1']],\n",
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
"\n",
" def createDataFrame(self, columns, data):\n",
" return pd.DataFrame.from_dict(data, columns = columns, orient = 'index')\n"
" def test_createDataFrameWithFirstDose(self):\n",
" # Given\n",
" vaersDescrs = [\n",
" {\n",
" 'VAERSDATA': self.createDataFrame(\n",
" index = [\"1048786\"],\n",
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
" data = [ ['Y', np.NaN, np.NaN]]),\n",
" 'VAERSVAX': self.createDataFrame(\n",
" index = [\"1048786\", \"1048786\"],\n",
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
" data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n",
" ['COVID19', 'MODERNA', '030L20A', '1']],\n",
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
" }\n",
" ]\n",
" \n",
" # When\n",
" dataFrame = _createDataFrame(vaersDescrs, \"MODERNA\", '1')\n",
" \n",
" # Then\n",
" dataFrameExpected = self.createDataFrame(\n",
" index = [\"1048786\"],\n",
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
" data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '030L20A', '1']],\n",
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
"\n",
" def test_createDataFrameWithSecondDose(self):\n",
" # Given\n",
" vaersDescrs = [\n",
" {\n",
" 'VAERSDATA': self.createDataFrame(\n",
" index = [\"1048786\"],\n",
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
" data = [ ['Y', np.NaN, np.NaN]]),\n",
" 'VAERSVAX': self.createDataFrame(\n",
" index = [\"1048786\", \"1048786\"],\n",
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
" data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n",
" ['COVID19', 'MODERNA', '030L20A', '1']],\n",
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
" }\n",
" ]\n",
" \n",
" # When\n",
" dataFrame = _createDataFrame(vaersDescrs, \"MODERNA\", '2')\n",
" \n",
" # Then\n",
" dataFrameExpected = self.createDataFrame(\n",
" index = [\"1048786\"],\n",
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
" data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '016M20A', '2']],\n",
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
"\n",
" def createDataFrame(self, index, columns, data, dtypes = {}):\n",
" return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n"
]
},
{
@@ -156,41 +217,40 @@
" def test_createBatchCodeTable2(self):\n",
" dataFrame = _createDataFrame(\n",
" [\n",
" {\n",
" {\n",
" 'VAERSDATA': self.createDataFrame(\n",
" index = [\"0916600\", \"0916601\"],\n",
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
" data = [ ['Y', np.NaN, np.NaN],\n",
" [np.NaN, np.NaN, 'Y']]),\n",
" 'VAERSVAX': self.createDataFrame(\n",
" index = [\"0916600\", \"0916601\"],\n",
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
" },\n",
" {\n",
" 'VAERSDATA': self.createDataFrame(\n",
" [ 'DIED', 'L_THREAT', 'DISABLE'],\n",
" {\n",
" '0916600': ['Y', np.NaN, np.NaN],\n",
" '0916601': [np.NaN, np.NaN, 'Y']\n",
" }),\n",
" index = [\"1996873\", \"1996874\"],\n",
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
" data = [ [np.NaN, np.NaN, np.NaN],\n",
" [np.NaN, np.NaN, 'Y']]),\n",
" 'VAERSVAX': self.createDataFrame(\n",
" [ 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n",
" {\n",
" '0916600': ['COVID19', 'MODERNA', '037K20A'],\n",
" '0916601': ['COVID19', 'MODERNA', '025L20A']\n",
" })\n",
" },\n",
" {\n",
" 'VAERSDATA': self.createDataFrame(\n",
" [ 'DIED', 'L_THREAT', 'DISABLE'],\n",
" {\n",
" '1996873': [np.NaN, np.NaN, np.NaN],\n",
" '1996874': [np.NaN, np.NaN, 'Y']\n",
" }),\n",
" 'VAERSVAX': self.createDataFrame(\n",
" [ 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n",
" {\n",
" '1996873': ['HPV9', 'MERCK & CO. INC.', 'R017624'],\n",
" '1996874': ['COVID19', 'MODERNA', '025L20A']\n",
" })\n",
" index = [\"1996873\", \"1996874\"],\n",
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
" data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n",
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
" }\n",
" ],\n",
" \"MODERNA\")\n",
" \"MODERNA\",\n",
" '1')\n",
"\n",
" self._test_createBatchCodeTable(dataFrame);\n",
"\n",
" def test_createBatchCodeTable(self):\n",
" self._test_createBatchCodeTable(createDataFrame(\"test/VAERS\", \"MODERNA\"));\n",
" self._test_createBatchCodeTable(createDataFrame(\"test/VAERS\", \"MODERNA\", '1'));\n",
"\n",
" def _test_createBatchCodeTable(self, dataFrame):\n",
" # When\n",
@@ -198,7 +258,7 @@
"\n",
" # Then\n",
" batchCodeTableExpected = pd.DataFrame(\n",
" data={\n",
" data = {\n",
" 'ADRs': [2, 1],\n",
" 'DEATHS': [0, 1],\n",
" 'DISABILITIES': [2, 0],\n",
@@ -207,8 +267,8 @@
" index = pd.MultiIndex.from_arrays([['025L20A', '037K20A']], names = ('VAX_LOT',)))\n",
" assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n",
"\n",
" def createDataFrame(self, columns, data):\n",
" return pd.DataFrame.from_dict(data, columns = columns, orient = 'index')\n"
" def createDataFrame(self, index, columns, data, dtypes = {}):\n",
" return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n"
]
},
{
@@ -229,7 +289,7 @@
"outputs": [],
"source": [
"def saveBatchCodeTable(manufacturer, excelFile):\n",
" batchCodeTable = createBatchCodeTable(createDataFrame(\"VAERS\", manufacturer))\n",
" batchCodeTable = createBatchCodeTable(createDataFrame(\"VAERS\", manufacturer, '1'))\n",
" display(manufacturer, batchCodeTable)\n",
" batchCodeTable.to_excel(excelFile)"
]