From c558c905139d13304eeb9bed8c436d8cf169c449 Mon Sep 17 00:00:00 2001 From: frankknoll Date: Sat, 29 Jan 2022 02:39:42 +0100 Subject: [PATCH] adding tests test_createDataFrameWithFirstDose() and test_createDataFrameWithSecondDose() --- HowBadIsMyBatch.ipynb | 200 +++++++++++++++++++++++++++--------------- 1 file changed, 130 insertions(+), 70 deletions(-) diff --git a/HowBadIsMyBatch.ipynb b/HowBadIsMyBatch.ipynb index 1981eb61594..9bacecce7e8 100644 --- a/HowBadIsMyBatch.ipynb +++ b/HowBadIsMyBatch.ipynb @@ -21,12 +21,18 @@ "metadata": {}, "outputs": [], "source": [ - "def _createDataFrame(vaersDescrs, manufacturer):\n", - " def vaersDescr2DataFrame(vaersDescr):\n", - " return pd.merge(vaersDescr['VAERSDATA'], vaersDescr['VAERSVAX'], left_index = True, right_index = True)\n", + "def _createDataFrame(vaersDescrs, manufacturer, dose):\n", + " def filter(df):\n", + " return df[(df[\"VAX_TYPE\"] == \"COVID19\") & (df[\"VAX_MANU\"] == manufacturer) & (df[\"VAX_DOSE_SERIES\"].str.contains(dose))]\n", "\n", - " df = pd.concat(map(vaersDescr2DataFrame, vaersDescrs))\n", - " return df[(df[\"VAX_TYPE\"] == \"COVID19\") & (df[\"VAX_MANU\"] == manufacturer)]" + " def vaersDescr2DataFrame(vaersDescr):\n", + " return pd.merge(\n", + " vaersDescr['VAERSDATA'],\n", + " filter(vaersDescr['VAERSVAX']),\n", + " left_index = True,\n", + " right_index = True)\n", + "\n", + " return pd.concat(map(vaersDescr2DataFrame, vaersDescrs))\n" ] }, { @@ -36,20 +42,21 @@ "metadata": {}, "outputs": [], "source": [ - "def createDataFrame(dataDir, manufacturer):\n", + "def createDataFrame(dataDir, manufacturer, dose):\n", " def readVaersDescr(year):\n", - " def read_csv(file, usecols):\n", - " return pd.read_csv(file, index_col = 'VAERS_ID', encoding = 'latin1', low_memory = False, usecols = usecols)\n", + " def read_csv(file, usecols, dtype = {}):\n", + " return pd.read_csv(file, index_col = 'VAERS_ID', encoding = 'latin1', low_memory = False, usecols = usecols, dtype = dtype)\n", "\n", " folder = dataDir + \"/\" + year + \"VAERSData/\"\n", " return {\n", " 'VAERSDATA': read_csv(folder + year + \"VAERSDATA.csv\", ['VAERS_ID', 'DIED', 'L_THREAT', 'DISABLE']),\n", - " 'VAERSVAX': read_csv(folder + year + \"VAERSVAX.csv\", ['VAERS_ID', 'VAX_DOSE_SERIES', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'])\n", + " 'VAERSVAX': read_csv(folder + year + \"VAERSVAX.csv\", ['VAERS_ID', 'VAX_DOSE_SERIES', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'], dtype = {\"VAX_DOSE_SERIES\": \"string\"})\n", " }\n", "\n", " return _createDataFrame(\n", " [readVaersDescr(\"2021\"), readVaersDescr(\"2022\")],\n", - " manufacturer)" + " manufacturer,\n", + " dose)" ] }, { @@ -97,49 +104,103 @@ " vaersDescrs = [\n", " {\n", " 'VAERSDATA': self.createDataFrame(\n", - " [ 'DIED', 'L_THREAT', 'DISABLE'],\n", - " {\n", - " '0916600': ['Y', np.NaN, np.NaN],\n", - " '0916601': [np.NaN, np.NaN, 'Y']\n", - " }),\n", + " index = [\"0916600\", \"0916601\"],\n", + " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", + " data = [ ['Y', np.NaN, np.NaN],\n", + " [np.NaN, np.NaN, 'Y']]),\n", " 'VAERSVAX': self.createDataFrame(\n", - " [ 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n", - " {\n", - " '0916600': ['COVID19', 'MODERNA', '037K20A'],\n", - " '0916601': ['COVID19', 'MODERNA', '025L20A']\n", - " })\n", + " index = [\"0916600\", \"0916601\"],\n", + " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", + " data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n", + " ['COVID19', 'MODERNA', '025L20A', '1']],\n", + " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " },\n", " {\n", " 'VAERSDATA': self.createDataFrame(\n", - " [ 'DIED', 'L_THREAT', 'DISABLE'],\n", - " {\n", - " '1996873': [np.NaN, np.NaN, np.NaN],\n", - " '1996874': [np.NaN, np.NaN, 'Y']\n", - " }),\n", + " index = [\"1996873\", \"1996874\"],\n", + " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", + " data = [ [np.NaN, np.NaN, np.NaN],\n", + " [np.NaN, np.NaN, 'Y']]),\n", " 'VAERSVAX': self.createDataFrame(\n", - " [ 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n", - " {\n", - " '1996873': ['HPV9', 'MERCK & CO. INC.', 'R017624'],\n", - " '1996874': ['COVID19', 'MODERNA', '025L20A']\n", - " })\n", + " index = [\"1996873\", \"1996874\"],\n", + " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", + " data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n", + " ['COVID19', 'MODERNA', '025L20A', '1']],\n", + " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " }\n", " ]\n", " \n", " # When\n", - " dataFrame = _createDataFrame(vaersDescrs, \"MODERNA\")\n", + " dataFrame = _createDataFrame(vaersDescrs, \"MODERNA\", '1')\n", " \n", " # Then\n", " dataFrameExpected = self.createDataFrame(\n", - " [ 'DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n", - " {\n", - " '0916600': ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '037K20A'],\n", - " '0916601': [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A'],\n", - " '1996874': [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A']\n", - " })\n", + " index = [\"0916600\", \"0916601\", \"1996874\"],\n", + " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", + " data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '037K20A', '1'],\n", + " [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A', '1'],\n", + " [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A', '1']],\n", + " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n", "\n", - " def createDataFrame(self, columns, data):\n", - " return pd.DataFrame.from_dict(data, columns = columns, orient = 'index')\n" + " def test_createDataFrameWithFirstDose(self):\n", + " # Given\n", + " vaersDescrs = [\n", + " {\n", + " 'VAERSDATA': self.createDataFrame(\n", + " index = [\"1048786\"],\n", + " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", + " data = [ ['Y', np.NaN, np.NaN]]),\n", + " 'VAERSVAX': self.createDataFrame(\n", + " index = [\"1048786\", \"1048786\"],\n", + " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", + " data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n", + " ['COVID19', 'MODERNA', '030L20A', '1']],\n", + " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", + " }\n", + " ]\n", + " \n", + " # When\n", + " dataFrame = _createDataFrame(vaersDescrs, \"MODERNA\", '1')\n", + " \n", + " # Then\n", + " dataFrameExpected = self.createDataFrame(\n", + " index = [\"1048786\"],\n", + " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", + " data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '030L20A', '1']],\n", + " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", + " assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n", + "\n", + " def test_createDataFrameWithSecondDose(self):\n", + " # Given\n", + " vaersDescrs = [\n", + " {\n", + " 'VAERSDATA': self.createDataFrame(\n", + " index = [\"1048786\"],\n", + " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", + " data = [ ['Y', np.NaN, np.NaN]]),\n", + " 'VAERSVAX': self.createDataFrame(\n", + " index = [\"1048786\", \"1048786\"],\n", + " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", + " data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n", + " ['COVID19', 'MODERNA', '030L20A', '1']],\n", + " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", + " }\n", + " ]\n", + " \n", + " # When\n", + " dataFrame = _createDataFrame(vaersDescrs, \"MODERNA\", '2')\n", + " \n", + " # Then\n", + " dataFrameExpected = self.createDataFrame(\n", + " index = [\"1048786\"],\n", + " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", + " data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '016M20A', '2']],\n", + " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", + " assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n", + "\n", + " def createDataFrame(self, index, columns, data, dtypes = {}):\n", + " return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n" ] }, { @@ -156,41 +217,40 @@ " def test_createBatchCodeTable2(self):\n", " dataFrame = _createDataFrame(\n", " [\n", - " {\n", + " {\n", + " 'VAERSDATA': self.createDataFrame(\n", + " index = [\"0916600\", \"0916601\"],\n", + " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", + " data = [ ['Y', np.NaN, np.NaN],\n", + " [np.NaN, np.NaN, 'Y']]),\n", + " 'VAERSVAX': self.createDataFrame(\n", + " index = [\"0916600\", \"0916601\"],\n", + " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", + " data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n", + " ['COVID19', 'MODERNA', '025L20A', '1']],\n", + " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", + " },\n", + " {\n", " 'VAERSDATA': self.createDataFrame(\n", - " [ 'DIED', 'L_THREAT', 'DISABLE'],\n", - " {\n", - " '0916600': ['Y', np.NaN, np.NaN],\n", - " '0916601': [np.NaN, np.NaN, 'Y']\n", - " }),\n", + " index = [\"1996873\", \"1996874\"],\n", + " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", + " data = [ [np.NaN, np.NaN, np.NaN],\n", + " [np.NaN, np.NaN, 'Y']]),\n", " 'VAERSVAX': self.createDataFrame(\n", - " [ 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n", - " {\n", - " '0916600': ['COVID19', 'MODERNA', '037K20A'],\n", - " '0916601': ['COVID19', 'MODERNA', '025L20A']\n", - " })\n", - " },\n", - " {\n", - " 'VAERSDATA': self.createDataFrame(\n", - " [ 'DIED', 'L_THREAT', 'DISABLE'],\n", - " {\n", - " '1996873': [np.NaN, np.NaN, np.NaN],\n", - " '1996874': [np.NaN, np.NaN, 'Y']\n", - " }),\n", - " 'VAERSVAX': self.createDataFrame(\n", - " [ 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n", - " {\n", - " '1996873': ['HPV9', 'MERCK & CO. INC.', 'R017624'],\n", - " '1996874': ['COVID19', 'MODERNA', '025L20A']\n", - " })\n", + " index = [\"1996873\", \"1996874\"],\n", + " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", + " data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n", + " ['COVID19', 'MODERNA', '025L20A', '1']],\n", + " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " }\n", " ],\n", - " \"MODERNA\")\n", + " \"MODERNA\",\n", + " '1')\n", "\n", " self._test_createBatchCodeTable(dataFrame);\n", "\n", " def test_createBatchCodeTable(self):\n", - " self._test_createBatchCodeTable(createDataFrame(\"test/VAERS\", \"MODERNA\"));\n", + " self._test_createBatchCodeTable(createDataFrame(\"test/VAERS\", \"MODERNA\", '1'));\n", "\n", " def _test_createBatchCodeTable(self, dataFrame):\n", " # When\n", @@ -198,7 +258,7 @@ "\n", " # Then\n", " batchCodeTableExpected = pd.DataFrame(\n", - " data={\n", + " data = {\n", " 'ADRs': [2, 1],\n", " 'DEATHS': [0, 1],\n", " 'DISABILITIES': [2, 0],\n", @@ -207,8 +267,8 @@ " index = pd.MultiIndex.from_arrays([['025L20A', '037K20A']], names = ('VAX_LOT',)))\n", " assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n", "\n", - " def createDataFrame(self, columns, data):\n", - " return pd.DataFrame.from_dict(data, columns = columns, orient = 'index')\n" + " def createDataFrame(self, index, columns, data, dtypes = {}):\n", + " return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n" ] }, { @@ -229,7 +289,7 @@ "outputs": [], "source": [ "def saveBatchCodeTable(manufacturer, excelFile):\n", - " batchCodeTable = createBatchCodeTable(createDataFrame(\"VAERS\", manufacturer))\n", + " batchCodeTable = createBatchCodeTable(createDataFrame(\"VAERS\", manufacturer, '1'))\n", " display(manufacturer, batchCodeTable)\n", " batchCodeTable.to_excel(excelFile)" ]