745 lines
32 KiB
Plaintext
745 lines
32 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9de5907f-18f5-4cb1-903e-26028ff1fa03",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"pd.set_option('display.max_rows', 100)\n",
|
|
"pd.set_option('display.max_columns', None)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b136967b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class DataFrameConverter:\n",
|
|
" \n",
|
|
" @staticmethod\n",
|
|
" def convertColumnsOfDataFrameToNumerics(dataFrame, columns):\n",
|
|
" for column in columns:\n",
|
|
" DataFrameConverter._convertColumnOfDataFrameToNumeric(dataFrame, column)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _convertColumnOfDataFrameToNumeric(dataFrame, column):\n",
|
|
" dataFrame[column] = np.where(dataFrame[column] == 'Y', 1, 0)\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a271254b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"class VaersDescrReader:\n",
|
|
" \n",
|
|
" def __init__(self, dataDir):\n",
|
|
" self.dataDir = dataDir \n",
|
|
"\n",
|
|
" def readAllVaersDescrs(self):\n",
|
|
" return self.readVaersDescrs([\"2021\", \"2022\"])\n",
|
|
" \n",
|
|
" def readVaersDescrs(self, years):\n",
|
|
" return [self.readVaersDescr(year) for year in years]\n",
|
|
"\n",
|
|
" def readVaersDescr(self, year):\n",
|
|
" folder = self.dataDir + \"/\" + year + \"VAERSData/\"\n",
|
|
" return {\n",
|
|
" 'VAERSDATA': self._readVAERSDATA(folder + year + \"VAERSDATA.csv\"),\n",
|
|
" 'VAERSVAX': self._readVAERSVAX(folder + year + \"VAERSVAX.csv\")\n",
|
|
" }\n",
|
|
"\n",
|
|
" def _readVAERSDATA(self, file):\n",
|
|
" VAERSDATA = self._read_csv(\n",
|
|
" file = file,\n",
|
|
" usecols = ['VAERS_ID', 'DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'])\n",
|
|
" DataFrameConverter.convertColumnsOfDataFrameToNumerics(\n",
|
|
" VAERSDATA,\n",
|
|
" ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'])\n",
|
|
" return VAERSDATA\n",
|
|
"\n",
|
|
" def _readVAERSVAX(self, file):\n",
|
|
" return self._read_csv(\n",
|
|
" file = file,\n",
|
|
" usecols = ['VAERS_ID', 'VAX_DOSE_SERIES', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n",
|
|
" dtype = {\"VAX_DOSE_SERIES\": \"string\"})\n",
|
|
"\n",
|
|
" def _read_csv(self, file, **kwargs):\n",
|
|
" return pd.read_csv(\n",
|
|
" file,\n",
|
|
" index_col = 'VAERS_ID',\n",
|
|
" encoding = 'latin1',\n",
|
|
" low_memory = False,\n",
|
|
" **kwargs)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7b5d6df0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"class VaersDescr2DataFrameConverter:\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createDataFrameFromDescr(vaersDescr):\n",
|
|
" return pd.merge(\n",
|
|
" vaersDescr['VAERSDATA'],\n",
|
|
" vaersDescr['VAERSVAX'],\n",
|
|
" how = 'left',\n",
|
|
" left_index = True,\n",
|
|
" right_index = True,\n",
|
|
" validate = 'one_to_many')\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createDataFrameFromDescrs(vaersDescrs):\n",
|
|
" dataFrames = [VaersDescr2DataFrameConverter.createDataFrameFromDescr(vaersDescr) for vaersDescr in vaersDescrs]\n",
|
|
" return pd.concat(dataFrames)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3ebcba86",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"class DataFrameFilter:\n",
|
|
" \n",
|
|
" def __init__(self, dataFrame):\n",
|
|
" self.dataFrame = dataFrame\n",
|
|
"\n",
|
|
" def filterBy(self, manufacturer = None, dose = None):\n",
|
|
" return self.dataFrame[self._isCovid19() & self._isManufacturer(manufacturer) & self._isDose(dose)]\n",
|
|
"\n",
|
|
" def filterForSevereEffects(self, dose):\n",
|
|
" return self.filterBy(dose = dose)\n",
|
|
"\n",
|
|
" def _isCovid19(self):\n",
|
|
" return self.dataFrame[\"VAX_TYPE\"] == \"COVID19\"\n",
|
|
"\n",
|
|
" def _isManufacturer(self, manufacturer):\n",
|
|
" return self.dataFrame[\"VAX_MANU\"] == manufacturer if manufacturer is not None else True\n",
|
|
"\n",
|
|
" def _isDose(self, dose):\n",
|
|
" return self.dataFrame[\"VAX_DOSE_SERIES\"].str.contains(dose) if dose is not None else True\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c62cfaff",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class AggregationHelper:\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def aggregateAndFlattenColumnsAndRenameColumns(dataFrame, aggFunctionsByColumn, columnNameMappingsDict):\n",
|
|
" aggregatedDataFrame = dataFrame.agg(aggFunctionsByColumn)\n",
|
|
" AggregationHelper._flattenColumns(aggregatedDataFrame)\n",
|
|
" return aggregatedDataFrame.rename(columns = columnNameMappingsDict)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _flattenColumns(dataFrame):\n",
|
|
" dataFrame.columns = [\"_\".join(a) for a in dataFrame.columns.to_flat_index()]\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "99945ca8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"class BatchCodeTableHelper:\n",
|
|
" \n",
|
|
" def __init__(self, dataFrame : pd.DataFrame):\n",
|
|
" self.dataFrame = dataFrame \n",
|
|
"\n",
|
|
" def createBatchCodeTable(self):\n",
|
|
" batchCodeTable = AggregationHelper.aggregateAndFlattenColumnsAndRenameColumns(\n",
|
|
" dataFrame = self.dataFrame.groupby('VAX_LOT'),\n",
|
|
" aggFunctionsByColumn = {\n",
|
|
" 'DIED': ['sum', 'size'],\n",
|
|
" 'L_THREAT': 'sum',\n",
|
|
" 'DISABLE': 'sum'\n",
|
|
" },\n",
|
|
" columnNameMappingsDict = {\n",
|
|
" \"DIED_size\": \"ADRs\",\n",
|
|
" \"DIED_sum\": \"DEATHS\",\n",
|
|
" \"L_THREAT_sum\": \"LIFE THREATENING ILLNESSES\",\n",
|
|
" \"DISABLE_sum\": \"DISABILITIES\"\n",
|
|
" })\n",
|
|
" batchCodeTable = batchCodeTable[['ADRs', 'DEATHS', 'DISABILITIES', 'LIFE THREATENING ILLNESSES']]\n",
|
|
" return batchCodeTable.sort_values(by = 'ADRs', ascending = False)\n",
|
|
"\n",
|
|
" # create table from https://www.howbadismybatch.com/combined.html\n",
|
|
" def createSevereEffectsBatchCodeTable(self):\n",
|
|
" batchCodeTable = AggregationHelper.aggregateAndFlattenColumnsAndRenameColumns(\n",
|
|
" dataFrame = self.dataFrame.groupby('VAX_LOT'),\n",
|
|
" aggFunctionsByColumn = {\n",
|
|
" 'DIED': ['sum', 'size'],\n",
|
|
" 'L_THREAT': 'sum',\n",
|
|
" 'DISABLE': 'sum',\n",
|
|
" 'HOSPITAL': 'sum',\n",
|
|
" 'ER_VISIT': 'sum'\n",
|
|
" },\n",
|
|
" columnNameMappingsDict = {\n",
|
|
" \"DIED_size\": \"ADRs\",\n",
|
|
" \"DIED_sum\": \"DEATHS\",\n",
|
|
" \"L_THREAT_sum\": \"LIFE THREATENING ILLNESSES\",\n",
|
|
" \"DISABLE_sum\": \"DISABILITIES\",\n",
|
|
" 'HOSPITAL_sum': 'HOSPITALISATIONS',\n",
|
|
" 'ER_VISIT_sum': 'EMERGENCY ROOM OR DOCTOR VISITS'\n",
|
|
" })\n",
|
|
" batchCodeTable = batchCodeTable[['ADRs', 'DEATHS', 'DISABILITIES', 'LIFE THREATENING ILLNESSES', 'HOSPITALISATIONS', 'EMERGENCY ROOM OR DOCTOR VISITS']]\n",
|
|
" batchCodeTable = batchCodeTable.sort_values(by = 'ADRs', ascending = False)\n",
|
|
" return self._addCompanyColumn(batchCodeTable, self._createCompanyByBatchCodeTable())\n",
|
|
"\n",
|
|
" def _addCompanyColumn(self, batchCodeTable, companyByBatchCodeTable):\n",
|
|
" return pd.merge(\n",
|
|
" batchCodeTable,\n",
|
|
" companyByBatchCodeTable,\n",
|
|
" how = 'left',\n",
|
|
" left_index = True,\n",
|
|
" right_index = True,\n",
|
|
" validate = 'one_to_one')\n",
|
|
"\n",
|
|
" def _createCompanyByBatchCodeTable(self):\n",
|
|
" return self._createManufacturerByBatchCodeTable().rename(columns = {\"VAX_MANU\": \"COMPANY\"})\n",
|
|
"\n",
|
|
" def _createManufacturerByBatchCodeTable(self):\n",
|
|
" manufacturerByBatchCodeTable = self.dataFrame[['VAX_LOT', 'VAX_MANU']]\n",
|
|
" manufacturerByBatchCodeTable = manufacturerByBatchCodeTable.drop_duplicates(subset = ['VAX_LOT'])\n",
|
|
" return manufacturerByBatchCodeTable.set_index('VAX_LOT')\n",
|
|
"\n",
|
|
"\n",
|
|
"class BatchCodeTableFactory:\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createBatchCodeTable(dataFrame : pd.DataFrame, manufacturer, dose):\n",
|
|
" filteredDataFrame = DataFrameFilter(dataFrame).filterBy(manufacturer = manufacturer, dose = dose)\n",
|
|
" return BatchCodeTableHelper(filteredDataFrame).createBatchCodeTable()\n",
|
|
"\n",
|
|
" # create table from https://www.howbadismybatch.com/combined.html\n",
|
|
" @staticmethod\n",
|
|
" def createSevereEffectsBatchCodeTable(dataFrame : pd.DataFrame, dose):\n",
|
|
" severeEffectsDataFrame = DataFrameFilter(dataFrame).filterForSevereEffects(dose)\n",
|
|
" return BatchCodeTableHelper(severeEffectsDataFrame).createSevereEffectsBatchCodeTable()\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "41d4fa30",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class DoseAnalysis:\n",
|
|
" \n",
|
|
" @staticmethod\n",
|
|
" def getDoseTable(dataFrame):\n",
|
|
" doseTable = AggregationHelper.aggregateAndFlattenColumnsAndRenameColumns(\n",
|
|
" dataFrame = dataFrame.groupby('VAX_DOSE_SERIES'),\n",
|
|
" aggFunctionsByColumn = {\n",
|
|
" 'DIED': ['sum', 'size'],\n",
|
|
" 'L_THREAT': 'sum',\n",
|
|
" 'DISABLE': 'sum'\n",
|
|
" },\n",
|
|
" columnNameMappingsDict = {\n",
|
|
" \"DIED_size\": \"Total reports\",\n",
|
|
" \"DIED_sum\": \"Deaths\",\n",
|
|
" \"L_THREAT_sum\": \"Life Threatening Illnesses\",\n",
|
|
" \"DISABLE_sum\": \"Disabilities\"\n",
|
|
" })\n",
|
|
" doseTable = doseTable[['Total reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses']]\n",
|
|
" doseTable['Severe reports (%)'] = (doseTable['Deaths'] + doseTable['Disabilities'] + doseTable['Life Threatening Illnesses']) / doseTable['Total reports'] * 100\n",
|
|
" return doseTable\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3dacedfd",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import unittest"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e59a1825",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class DataFrameFilterTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_filterBy(self):\n",
|
|
" # Given\n",
|
|
" dataFrameFilter = DataFrameFilter(\n",
|
|
" VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [1, 0, 0],\n",
|
|
" [0, 0, 1]],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" },\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [0, 0, 0],\n",
|
|
" [0, 0, 1]],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ]))\n",
|
|
" \n",
|
|
" # When\n",
|
|
" dataFrame = dataFrameFilter.filterBy(manufacturer = \"MODERNA\", dose = '1')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" [0, 0, 1, 'COVID19', 'MODERNA', '025L20A', '1'],\n",
|
|
" [0, 0, 1, 'COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_filterForSevereEffects(self):\n",
|
|
" # Given\n",
|
|
" dataFrameFilter = DataFrameFilter(\n",
|
|
" VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 1, 0, 1, 1],\n",
|
|
" [0, 0, 1, 0, 1]],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'PFIZER\\BIONTECH', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ]))\n",
|
|
"\n",
|
|
" # When\n",
|
|
" dataFrame = dataFrameFilter.filterForSevereEffects(dose = '1')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [1, 1, 0, 1, 1, 'COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" [0, 0, 1, 0, 1, 'COVID19', 'PFIZER\\BIONTECH', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_filterByFirstDose(self):\n",
|
|
" # Given\n",
|
|
" dataFrameFilter = DataFrameFilter(\n",
|
|
" VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1048786\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n",
|
|
" ['COVID19', 'MODERNA', '030L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ]))\n",
|
|
" \n",
|
|
" # When\n",
|
|
" dataFrame = dataFrameFilter.filterBy(manufacturer = \"MODERNA\", dose = '1')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_filterBySecondDose(self):\n",
|
|
" # Given\n",
|
|
" dataFrameFilter = DataFrameFilter(\n",
|
|
" VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1048786\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n",
|
|
" ['COVID19', 'MODERNA', '030L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ]))\n",
|
|
"\n",
|
|
" # When\n",
|
|
" dataFrame = dataFrameFilter.filterBy(manufacturer = \"MODERNA\", dose = '2')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '016M20A', '2']],\n",
|
|
" index = [\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def createDataFrame(self, index, columns, data, dtypes = {}):\n",
|
|
" return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e14465d7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class BatchCodeTableFactoryTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_createSevereEffectsBatchCodeTable(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 1, 0, 1, 1],\n",
|
|
" [0, 0, 1, 0, 1]],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'PFIZER\\BIONTECH', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ])\n",
|
|
"\n",
|
|
" # When\n",
|
|
" batchCodeTable = BatchCodeTableFactory.createSevereEffectsBatchCodeTable(dataFrame, '1')\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" batchCodeTableExpected = pd.DataFrame(\n",
|
|
" data = {\n",
|
|
" 'ADRs': [1, 1],\n",
|
|
" 'DEATHS': [0, 1],\n",
|
|
" 'DISABILITIES': [1, 0],\n",
|
|
" 'LIFE THREATENING ILLNESSES': [0, 1],\n",
|
|
" 'HOSPITALISATIONS': [0, 1],\n",
|
|
" 'EMERGENCY ROOM OR DOCTOR VISITS': [1, 1],\n",
|
|
" 'COMPANY': ['PFIZER\\BIONTECH', 'MODERNA']\n",
|
|
" },\n",
|
|
" index = pd.Index(['025L20A', '037K20A'], name = 'VAX_LOT'))\n",
|
|
" assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_createBatchCodeTable2(self):\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 0, 0, 0, 0],\n",
|
|
" [0, 0, 1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" },\n",
|
|
" {\n",
|
|
" 'VAERSDATA': self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"]),\n",
|
|
" 'VAERSVAX': self.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ])\n",
|
|
" self._test_createBatchCodeTable(dataFrame, \"MODERNA\", '1')\n",
|
|
"\n",
|
|
" def test_createBatchCodeTable(self):\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" VaersDescrReader(dataDir = \"test/VAERS\").readAllVaersDescrs())\n",
|
|
" self._test_createBatchCodeTable(dataFrame, \"MODERNA\", '1')\n",
|
|
"\n",
|
|
" def _test_createBatchCodeTable(self, dataFrame, manufacturer, dose):\n",
|
|
" # When\n",
|
|
" batchCodeTable = BatchCodeTableFactory.createBatchCodeTable(dataFrame, manufacturer, dose)\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" batchCodeTableExpected = pd.DataFrame(\n",
|
|
" data = {\n",
|
|
" 'ADRs': [2, 1],\n",
|
|
" 'DEATHS': [0, 1],\n",
|
|
" 'DISABILITIES': [2, 0],\n",
|
|
" 'LIFE THREATENING ILLNESSES': [0, 0]\n",
|
|
" },\n",
|
|
" index = pd.Index(['025L20A', '037K20A'], name = 'VAX_LOT'))\n",
|
|
" assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def createDataFrame(self, index, columns, data, dtypes = {}):\n",
|
|
" return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "44c121ec",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class DoseAnalysisTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_getDoseTable(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = self.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '016M20A', '2'],\n",
|
|
" [1, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1'],\n",
|
|
" [1, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\",\n",
|
|
" \"4711\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" \n",
|
|
" # When\n",
|
|
" doseTable = DoseAnalysis.getDoseTable(dataFrame)\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" assert_frame_equal(\n",
|
|
" doseTable,\n",
|
|
" pd.DataFrame(\n",
|
|
" data = {\n",
|
|
" 'Total reports': [2, 1],\n",
|
|
" 'Deaths': [2, 1],\n",
|
|
" 'Disabilities': [1, 0],\n",
|
|
" 'Life Threatening Illnesses': [1, 0],\n",
|
|
" 'Severe reports (%)': [(2 + 1 + 1)/2 * 100, (1 + 0 + 0)/1 * 100]\n",
|
|
" },\n",
|
|
" index = pd.Index(['1', '2'], dtype = \"string\", name = 'VAX_DOSE_SERIES')))\n",
|
|
" \n",
|
|
" def createDataFrame(self, index, columns, data, dtypes = {}):\n",
|
|
" return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5a8bff1b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"unittest.main(argv = [''], verbosity = 2, exit = False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "86e0e4f2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def saveBatchCodeTable(manufacturer, excelFile):\n",
|
|
" vaersDescrs = VaersDescrReader(dataDir = \"VAERS\").readAllVaersDescrs()\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(vaersDescrs)\n",
|
|
" batchCodeTable = BatchCodeTableFactory.createBatchCodeTable(dataFrame, manufacturer = manufacturer, dose = '1')\n",
|
|
" display(manufacturer + ':', batchCodeTable)\n",
|
|
" batchCodeTable.to_excel(excelFile)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ab170c16",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"saveBatchCodeTable(\"MODERNA\", \"results/moderna.xlsx\")\n",
|
|
"saveBatchCodeTable(\"PFIZER\\BIONTECH\", \"results/pfizer.xlsx\")\n",
|
|
"saveBatchCodeTable(\"JANSSEN\", \"results/janssen.xlsx\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "bc56831d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def saveSevereEffectsBatchCodeTable(excelFile):\n",
|
|
" vaersDescrs = VaersDescrReader(dataDir = \"VAERS\").readAllVaersDescrs()\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(vaersDescrs)\n",
|
|
" severeEffectsBatchCodeTable = BatchCodeTableFactory.createSevereEffectsBatchCodeTable(dataFrame, dose = '1')\n",
|
|
" display('severeEffectsBatchCodeTable:', severeEffectsBatchCodeTable)\n",
|
|
" severeEffectsBatchCodeTable.to_excel(excelFile)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ace3fed9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"saveSevereEffectsBatchCodeTable('results/severeEffects.xlsx')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "1b228a16",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Variation in Effect of First and Second Doses"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "202f7c3f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# https://www.howbadismybatch.com/firstsecond.html\n",
|
|
"\n",
|
|
"def getDoseTable():\n",
|
|
" vaersDescrs = VaersDescrReader(dataDir = \"VAERS\").readAllVaersDescrs()\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(vaersDescrs)\n",
|
|
" return DoseAnalysis.getDoseTable(dataFrame)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "394fa19d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"getDoseTable()"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|