{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "9de5907f-18f5-4cb1-903e-26028ff1fa03", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "\n", "pd.set_option('display.max_rows', 100)\n", "pd.set_option('display.max_columns', None)" ] }, { "cell_type": "code", "execution_count": null, "id": "7b5d6df0", "metadata": {}, "outputs": [], "source": [ "def createDataFrameFromDescr(vaersDescr, manufacturer, dose):\n", " def filter(df):\n", " return df[\n", " (df[\"VAX_TYPE\"] == \"COVID19\") &\n", " (df[\"VAX_MANU\"] == manufacturer) &\n", " (df[\"VAX_DOSE_SERIES\"].str.contains(dose))]\n", " \n", " return pd.merge(\n", " vaersDescr['VAERSDATA'],\n", " filter(vaersDescr['VAERSVAX']),\n", " left_index = True,\n", " right_index = True)\n", "\n", "def createDataFrameFromDescrs(vaersDescrs, manufacturer, dose):\n", " _createDataFrameFromDescr = lambda vaersDescr: createDataFrameFromDescr(vaersDescr, manufacturer, dose)\n", " dataFrames = map(_createDataFrameFromDescr, vaersDescrs)\n", " return pd.concat(dataFrames)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "233bc590", "metadata": {}, "outputs": [], "source": [ "def createDataFrameFromFiles(dataDir, manufacturer, dose):\n", " def readVaersDescr(year):\n", " def read_csv(file, usecols, dtype = {}):\n", " return pd.read_csv(\n", " file,\n", " index_col = 'VAERS_ID',\n", " encoding = 'latin1',\n", " low_memory = False,\n", " usecols = usecols,\n", " dtype = dtype)\n", "\n", " folder = dataDir + \"/\" + year + \"VAERSData/\"\n", " return {\n", " 'VAERSDATA':\n", " read_csv(\n", " folder + year + \"VAERSDATA.csv\",\n", " ['VAERS_ID', 'DIED', 'L_THREAT', 'DISABLE']),\n", " 'VAERSVAX':\n", " read_csv(\n", " folder + year + \"VAERSVAX.csv\",\n", " ['VAERS_ID', 'VAX_DOSE_SERIES', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n", " dtype = {\"VAX_DOSE_SERIES\": \"string\"})\n", " }\n", "\n", " return createDataFrameFromDescrs(\n", " [readVaersDescr(\"2021\"), readVaersDescr(\"2022\")],\n", " manufacturer,\n", " dose)" ] }, { "cell_type": "code", "execution_count": null, "id": "99945ca8", "metadata": {}, "outputs": [], "source": [ "def createBatchCodeTable(df : pd.DataFrame):\n", " def filter(df, col):\n", " return df[df[col] == 'Y'][['VAX_LOT']]\n", "\n", " batchCodeTableDict = {\n", " 'ADRs': df[['VAX_LOT']].value_counts(),\n", " 'DEATHS': filter(df, 'DIED').value_counts(),\n", " 'DISABILITIES': filter(df, 'DISABLE').value_counts(),\n", " 'LIFE THREATENING ILLNESSES': filter(df, 'L_THREAT').value_counts()\n", " }\n", " return pd.concat(batchCodeTableDict, axis = 1).replace(to_replace = np.nan, value = 0)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3dacedfd", "metadata": {}, "outputs": [], "source": [ "import unittest" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from pandas.testing import assert_frame_equal\n", "\n", "class CreateDataFrameTest(unittest.TestCase):\n", "\n", " def test_createDataFrameFromDescrs(self):\n", " # Given\n", " vaersDescrs = [\n", " {\n", " 'VAERSDATA': self.createDataFrame(\n", " index = [\"0916600\", \"0916601\"],\n", " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", " data = [ ['Y', np.NaN, np.NaN],\n", " [np.NaN, np.NaN, 'Y']]),\n", " 'VAERSVAX': self.createDataFrame(\n", " index = [\"0916600\", \"0916601\"],\n", " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n", " ['COVID19', 'MODERNA', '025L20A', '1']],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " },\n", " {\n", " 'VAERSDATA': self.createDataFrame(\n", " index = [\"1996873\", \"1996874\"],\n", " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", " data = [ [np.NaN, np.NaN, np.NaN],\n", " [np.NaN, np.NaN, 'Y']]),\n", " 'VAERSVAX': self.createDataFrame(\n", " index = [\"1996873\", \"1996874\"],\n", " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n", " ['COVID19', 'MODERNA', '025L20A', '1']],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " }\n", " ]\n", " \n", " # When\n", " dataFrame = createDataFrameFromDescrs(vaersDescrs, \"MODERNA\", '1')\n", " \n", " # Then\n", " dataFrameExpected = self.createDataFrame(\n", " index = [\"0916600\", \"0916601\", \"1996874\"],\n", " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '037K20A', '1'],\n", " [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A', '1'],\n", " [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A', '1']],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n", "\n", " def test_createDataFrameFromDescrsWithFirstDose(self):\n", " # Given\n", " vaersDescrs = [\n", " {\n", " 'VAERSDATA': self.createDataFrame(\n", " index = [\"1048786\"],\n", " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", " data = [ ['Y', np.NaN, np.NaN]]),\n", " 'VAERSVAX': self.createDataFrame(\n", " index = [\"1048786\", \"1048786\"],\n", " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n", " ['COVID19', 'MODERNA', '030L20A', '1']],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " }\n", " ]\n", " \n", " # When\n", " dataFrame = createDataFrameFromDescrs(vaersDescrs, \"MODERNA\", '1')\n", " \n", " # Then\n", " dataFrameExpected = self.createDataFrame(\n", " index = [\"1048786\"],\n", " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '030L20A', '1']],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n", "\n", " def test_createDataFrameFromDescrsWithSecondDose(self):\n", " # Given\n", " vaersDescrs = [\n", " {\n", " 'VAERSDATA': self.createDataFrame(\n", " index = [\"1048786\"],\n", " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", " data = [ ['Y', np.NaN, np.NaN]]),\n", " 'VAERSVAX': self.createDataFrame(\n", " index = [\"1048786\", \"1048786\"],\n", " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n", " ['COVID19', 'MODERNA', '030L20A', '1']],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " }\n", " ]\n", " \n", " # When\n", " dataFrame = createDataFrameFromDescrs(vaersDescrs, \"MODERNA\", '2')\n", " \n", " # Then\n", " dataFrameExpected = self.createDataFrame(\n", " index = [\"1048786\"],\n", " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '016M20A', '2']],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n", "\n", " def createDataFrame(self, index, columns, data, dtypes = {}):\n", " return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e14465d7", "metadata": {}, "outputs": [], "source": [ "from pandas.testing import assert_frame_equal\n", "\n", "class BatchCodeTableTest(unittest.TestCase):\n", "\n", " def test_createBatchCodeTable2(self):\n", " dataFrame = createDataFrameFromDescrs(\n", " [\n", " {\n", " 'VAERSDATA': self.createDataFrame(\n", " index = [\"0916600\", \"0916601\"],\n", " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", " data = [ ['Y', np.NaN, np.NaN],\n", " [np.NaN, np.NaN, 'Y']]),\n", " 'VAERSVAX': self.createDataFrame(\n", " index = [\"0916600\", \"0916601\"],\n", " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n", " ['COVID19', 'MODERNA', '025L20A', '1']],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " },\n", " {\n", " 'VAERSDATA': self.createDataFrame(\n", " index = [\"1996873\", \"1996874\"],\n", " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", " data = [ [np.NaN, np.NaN, np.NaN],\n", " [np.NaN, np.NaN, 'Y']]),\n", " 'VAERSVAX': self.createDataFrame(\n", " index = [\"1996873\", \"1996874\"],\n", " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n", " ['COVID19', 'MODERNA', '025L20A', '1']],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " }\n", " ],\n", " \"MODERNA\",\n", " '1')\n", "\n", " self._test_createBatchCodeTable(dataFrame);\n", "\n", " def test_createBatchCodeTable(self):\n", " self._test_createBatchCodeTable(createDataFrameFromFiles(\"test/VAERS\", \"MODERNA\", '1'));\n", "\n", " def _test_createBatchCodeTable(self, dataFrame):\n", " # When\n", " batchCodeTable = createBatchCodeTable(dataFrame)\n", "\n", " # Then\n", " batchCodeTableExpected = pd.DataFrame(\n", " data = {\n", " 'ADRs': [2, 1],\n", " 'DEATHS': [0, 1],\n", " 'DISABILITIES': [2, 0],\n", " 'LIFE THREATENING ILLNESSES': [0, 0]\n", " },\n", " index = pd.MultiIndex.from_arrays([['025L20A', '037K20A']], names = ('VAX_LOT',)))\n", " assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n", "\n", " def createDataFrame(self, index, columns, data, dtypes = {}):\n", " return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5a8bff1b", "metadata": {}, "outputs": [], "source": [ "unittest.main(argv = [''], verbosity = 2, exit = False)" ] }, { "cell_type": "code", "execution_count": null, "id": "86e0e4f2", "metadata": {}, "outputs": [], "source": [ "def saveBatchCodeTable(manufacturer, excelFile):\n", " batchCodeTable = createBatchCodeTable(createDataFrameFromFiles(\"VAERS\", manufacturer, '1'))\n", " display(manufacturer, batchCodeTable)\n", " batchCodeTable.to_excel(excelFile)" ] }, { "cell_type": "code", "execution_count": null, "id": "ab170c16", "metadata": {}, "outputs": [], "source": [ "saveBatchCodeTable(\"MODERNA\", \"results/moderna.xlsx\")\n", "saveBatchCodeTable(\"PFIZER\\BIONTECH\", \"results/pfizer.xlsx\")\n", "saveBatchCodeTable(\"JANSSEN\", \"results/janssen.xlsx\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.7" } }, "nbformat": 4, "nbformat_minor": 5 }