426 lines
18 KiB
Plaintext
426 lines
18 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9de5907f-18f5-4cb1-903e-26028ff1fa03",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"pd.set_option('display.max_rows', 100)\n",
|
|
"pd.set_option('display.max_columns', None)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7b5d6df0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def createDataFrameFromDescr(vaersDescr):\n",
|
|
" return pd.merge(\n",
|
|
" vaersDescr['VAERSDATA'],\n",
|
|
" vaersDescr['VAERSVAX'],\n",
|
|
" how = 'left',\n",
|
|
" left_index = True,\n",
|
|
" right_index = True,\n",
|
|
" validate = 'one_to_many')\n",
|
|
"\n",
|
|
"def createDataFrameFromDescrs(vaersDescrs):\n",
|
|
" dataFrames = map(createDataFrameFromDescr, vaersDescrs)\n",
|
|
" return pd.concat(dataFrames)\n",
|
|
"\n",
|
|
"def createAndFilterDataFrameFromDescrs(vaersDescrs, manufacturer, dose):\n",
|
|
" def filterDataFrame(df):\n",
|
|
" return df[\n",
|
|
" (df[\"VAX_TYPE\"] == \"COVID19\") &\n",
|
|
" (df[\"VAX_MANU\"] == manufacturer) &\n",
|
|
" (df[\"VAX_DOSE_SERIES\"].str.contains(dose))]\n",
|
|
" \n",
|
|
" return filterDataFrame(createDataFrameFromDescrs(vaersDescrs))\n",
|
|
"\n",
|
|
"def createDataFrameSevereEffectsFromDescrs(vaersDescrs, dose):\n",
|
|
" def filterDataFrame(df):\n",
|
|
" return df[\n",
|
|
" (df[\"VAX_TYPE\"] == \"COVID19\") &\n",
|
|
" (df[\"VAX_DOSE_SERIES\"].str.contains(dose))]\n",
|
|
"\n",
|
|
" return filterDataFrame(createDataFrameFromDescrs(vaersDescrs))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "233bc590",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def read_csv(file, usecols, dtype = {}):\n",
|
|
" return pd.read_csv(\n",
|
|
" file,\n",
|
|
" index_col = 'VAERS_ID',\n",
|
|
" encoding = 'latin1',\n",
|
|
" low_memory = False,\n",
|
|
" usecols = usecols,\n",
|
|
" dtype = dtype)\n",
|
|
"\n",
|
|
"def readVaersDescr(dataDir, year):\n",
|
|
" folder = dataDir + \"/\" + year + \"VAERSData/\"\n",
|
|
" return {\n",
|
|
" 'VAERSDATA':\n",
|
|
" read_csv(\n",
|
|
" folder + year + \"VAERSDATA.csv\",\n",
|
|
" ['VAERS_ID', 'DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT']),\n",
|
|
" 'VAERSVAX':\n",
|
|
" read_csv(\n",
|
|
" folder + year + \"VAERSVAX.csv\",\n",
|
|
" ['VAERS_ID', 'VAX_DOSE_SERIES', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n",
|
|
" dtype = {\"VAX_DOSE_SERIES\": \"string\"})\n",
|
|
" }\n",
|
|
"\n",
|
|
"def createAndFilterDataFrameFromFiles(dataDir, manufacturer, dose):\n",
|
|
" return createAndFilterDataFrameFromDescrs(\n",
|
|
" [readVaersDescr(dataDir, \"2021\"), readVaersDescr(dataDir, \"2022\")],\n",
|
|
" manufacturer,\n",
|
|
" dose)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "99945ca8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def createBatchCodeTable(df : pd.DataFrame):\n",
|
|
" def filter(df, col):\n",
|
|
" return df[df[col] == 'Y'][['VAX_LOT']]\n",
|
|
"\n",
|
|
" batchCodeTableDict = {\n",
|
|
" 'ADRs': df[['VAX_LOT']].value_counts(),\n",
|
|
" 'DEATHS': filter(df, 'DIED').value_counts(),\n",
|
|
" 'DISABILITIES': filter(df, 'DISABLE').value_counts(),\n",
|
|
" 'LIFE THREATENING ILLNESSES': filter(df, 'L_THREAT').value_counts()\n",
|
|
" }\n",
|
|
" return pd.concat(batchCodeTableDict, axis = 'columns').replace(to_replace = np.nan, value = 0)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3dacedfd",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import unittest"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e59a1825",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class CreateAndFilterDataFrameTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_createAndFilterDataFrameFromDescrs(self):\n",
|
|
" # Given\n",
|
|
" vaersDescrs = [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ ['Y', np.NaN, np.NaN],\n",
|
|
" [np.NaN, np.NaN, 'Y']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" },\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [np.NaN, np.NaN, np.NaN],\n",
|
|
" [np.NaN, np.NaN, 'Y']],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ]\n",
|
|
" \n",
|
|
" # When\n",
|
|
" dataFrame = createAndFilterDataFrameFromDescrs(vaersDescrs, \"MODERNA\", '1')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A', '1'],\n",
|
|
" [np.NaN, np.NaN, 'Y', 'COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_createDataFrameFromForSevereEffects(self):\n",
|
|
" # Given\n",
|
|
" vaersDescrs = [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ ['Y', 'Y', np.NaN, 'Y', 'Y'],\n",
|
|
" [np.NaN, np.NaN, 'Y', np.NaN, 'Y']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'PFIZER\\BIONTECH', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ]\n",
|
|
" \n",
|
|
" # When\n",
|
|
" dataFrame = createDataFrameSevereEffectsFromDescrs(vaersDescrs, '1')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['Y', 'Y', np.NaN, 'Y', 'Y', 'COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" [np.NaN, np.NaN, 'Y', np.NaN, 'Y', 'COVID19', 'PFIZER\\BIONTECH', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_createAndFilterDataFrameFromDescrsWithFirstDose(self):\n",
|
|
" # Given\n",
|
|
" vaersDescrs = [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ ['Y', np.NaN, np.NaN]],\n",
|
|
" index = [\n",
|
|
" \"1048786\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n",
|
|
" ['COVID19', 'MODERNA', '030L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ]\n",
|
|
" \n",
|
|
" # When\n",
|
|
" dataFrame = createAndFilterDataFrameFromDescrs(vaersDescrs, \"MODERNA\", '1')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '030L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_createAndFilterDataFrameFromDescrsWithSecondDose(self):\n",
|
|
" # Given\n",
|
|
" vaersDescrs = [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ ['Y', np.NaN, np.NaN]],\n",
|
|
" index = [\n",
|
|
" \"1048786\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n",
|
|
" ['COVID19', 'MODERNA', '030L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ]\n",
|
|
" \n",
|
|
" # When\n",
|
|
" dataFrame = createAndFilterDataFrameFromDescrs(vaersDescrs, \"MODERNA\", '2')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['Y', np.NaN, np.NaN, 'COVID19', 'MODERNA', '016M20A', '2']],\n",
|
|
" index = [\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def createDataFrame(self, index, columns, data, dtypes = {}):\n",
|
|
" return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e14465d7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class BatchCodeTableTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_createBatchCodeTable2(self):\n",
|
|
" dataFrame = createAndFilterDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ ['Y', np.NaN, np.NaN],\n",
|
|
" [np.NaN, np.NaN, 'Y']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" },\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [np.NaN, np.NaN, np.NaN],\n",
|
|
" [np.NaN, np.NaN, 'Y']],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ],\n",
|
|
" \"MODERNA\",\n",
|
|
" '1')\n",
|
|
"\n",
|
|
" self._test_createBatchCodeTable(dataFrame);\n",
|
|
"\n",
|
|
" def test_createBatchCodeTable(self):\n",
|
|
" self._test_createBatchCodeTable(createAndFilterDataFrameFromFiles(\"test/VAERS\", \"MODERNA\", '1'));\n",
|
|
"\n",
|
|
" def _test_createBatchCodeTable(self, dataFrame):\n",
|
|
" # When\n",
|
|
" batchCodeTable = createBatchCodeTable(dataFrame)\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" batchCodeTableExpected = pd.DataFrame(\n",
|
|
" data = {\n",
|
|
" 'ADRs': [2, 1],\n",
|
|
" 'DEATHS': [0, 1],\n",
|
|
" 'DISABILITIES': [2, 0],\n",
|
|
" 'LIFE THREATENING ILLNESSES': [0, 0]\n",
|
|
" },\n",
|
|
" index = pd.MultiIndex.from_arrays([['025L20A', '037K20A']], names = ('VAX_LOT',)))\n",
|
|
" assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def createDataFrame(self, index, columns, data, dtypes = {}):\n",
|
|
" return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5a8bff1b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"unittest.main(argv = [''], verbosity = 2, exit = False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "86e0e4f2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def saveBatchCodeTable(manufacturer, excelFile):\n",
|
|
" batchCodeTable = createBatchCodeTable(createAndFilterDataFrameFromFiles(\"VAERS\", manufacturer, '1'))\n",
|
|
" display(manufacturer, batchCodeTable)\n",
|
|
" batchCodeTable.to_excel(excelFile)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ab170c16",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"saveBatchCodeTable(\"MODERNA\", \"results/moderna.xlsx\")\n",
|
|
"saveBatchCodeTable(\"PFIZER\\BIONTECH\", \"results/pfizer.xlsx\")\n",
|
|
"saveBatchCodeTable(\"JANSSEN\", \"results/janssen.xlsx\")"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|