1570 lines
67 KiB
Plaintext
1570 lines
67 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9de5907f-18f5-4cb1-903e-26028ff1fa03",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"pd.set_option('display.max_rows', 100)\n",
|
|
"pd.set_option('display.max_columns', None)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a271254b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"class VaersDescrReader:\n",
|
|
" \n",
|
|
" def __init__(self, dataDir):\n",
|
|
" self.dataDir = dataDir \n",
|
|
"\n",
|
|
" def readVaersDescrs(self, years):\n",
|
|
" return [self.readVaersDescr(year) for year in years]\n",
|
|
"\n",
|
|
" def readVaersDescr(self, year):\n",
|
|
" return {\n",
|
|
" 'VAERSDATA': self._readVAERSDATA(self.dataDir + \"/\" + year + \"VAERSDATA.csv\"),\n",
|
|
" 'VAERSVAX': self._readVAERSVAX(self.dataDir + \"/\" + year + \"VAERSVAX.csv\")\n",
|
|
" }\n",
|
|
"\n",
|
|
" def readNonDomesticVaersDescr(self):\n",
|
|
" return {\n",
|
|
" 'VAERSDATA': self._readVAERSDATA(self.dataDir + \"/\" + \"NonDomesticVAERSDATA.csv\"),\n",
|
|
" 'VAERSVAX': self._readVAERSVAX(self.dataDir + \"/\" + \"NonDomesticVAERSVAX.csv\")\n",
|
|
" }\n",
|
|
"\n",
|
|
" def _readVAERSDATA(self, file):\n",
|
|
" return self._read_csv(\n",
|
|
" file = file,\n",
|
|
" usecols = ['VAERS_ID', 'RECVDATE', 'DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT', 'SPLTTYPE'],\n",
|
|
" parse_dates = ['RECVDATE'],\n",
|
|
" date_parser = lambda dateStr: pd.to_datetime(dateStr, format = \"%m/%d/%Y\"))\n",
|
|
"\n",
|
|
" def _readVAERSVAX(self, file):\n",
|
|
" return self._read_csv(\n",
|
|
" file = file,\n",
|
|
" usecols = ['VAERS_ID', 'VAX_DOSE_SERIES', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n",
|
|
" dtype = {\"VAX_DOSE_SERIES\": \"string\"})\n",
|
|
"\n",
|
|
" def _read_csv(self, file, **kwargs):\n",
|
|
" return pd.read_csv(\n",
|
|
" file,\n",
|
|
" index_col = 'VAERS_ID',\n",
|
|
" encoding = 'latin1',\n",
|
|
" low_memory = False,\n",
|
|
" **kwargs)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7b5d6df0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"class VaersDescr2DataFrameConverter:\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createDataFrameFromDescr(vaersDescr):\n",
|
|
" return pd.merge(\n",
|
|
" vaersDescr['VAERSDATA'],\n",
|
|
" vaersDescr['VAERSVAX'],\n",
|
|
" how = 'left',\n",
|
|
" left_index = True,\n",
|
|
" right_index = True,\n",
|
|
" validate = 'one_to_many')\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createDataFrameFromDescrs(vaersDescrs):\n",
|
|
" dataFrames = [VaersDescr2DataFrameConverter.createDataFrameFromDescr(vaersDescr) for vaersDescr in vaersDescrs]\n",
|
|
" return pd.concat(dataFrames)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6b639196",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class DataFrameNormalizer:\n",
|
|
" \n",
|
|
" @staticmethod\n",
|
|
" def normalize(dataFrame):\n",
|
|
" DataFrameNormalizer.convertVAX_LOTColumnToUpperCase(dataFrame)\n",
|
|
" DataFrameNormalizer._convertColumnsOfDataFrame_Y_to_1_else_0(\n",
|
|
" dataFrame,\n",
|
|
" ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'])\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def convertVAX_LOTColumnToUpperCase(dataFrame):\n",
|
|
" dataFrame['VAX_LOT'] = dataFrame['VAX_LOT'].str.upper()\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _convertColumnsOfDataFrame_Y_to_1_else_0(dataFrame, columns):\n",
|
|
" for column in columns:\n",
|
|
" DataFrameNormalizer._convertColumnOfDataFrame_Y_to_1_else_0(dataFrame, column)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _convertColumnOfDataFrame_Y_to_1_else_0(dataFrame, column):\n",
|
|
" dataFrame[column] = DataFrameNormalizer._where(\n",
|
|
" condition = dataFrame[column] == 'Y',\n",
|
|
" trueValue = 1,\n",
|
|
" falseValue = 0)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _where(condition, trueValue, falseValue):\n",
|
|
" return np.where(condition, trueValue, falseValue) \n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3ebcba86",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"class DataFrameFilter:\n",
|
|
" \n",
|
|
" def filterByCovid19(self, dataFrame):\n",
|
|
" return dataFrame[self._isCovid19(dataFrame)]\n",
|
|
"\n",
|
|
" def filterByFlu(self, dataFrame):\n",
|
|
" return dataFrame[self._isFlu(dataFrame)]\n",
|
|
"\n",
|
|
" def filterByCountry(self, dataFrame, country, countryColumnName):\n",
|
|
" return dataFrame[dataFrame[countryColumnName] == country]\n",
|
|
"\n",
|
|
" def filterBy(self, dataFrame, manufacturer = None, dose = None):\n",
|
|
" return dataFrame[self._isManufacturer(dataFrame, manufacturer) & self._isDose(dataFrame, dose)]\n",
|
|
"\n",
|
|
" def _isCovid19(self, dataFrame):\n",
|
|
" return dataFrame[\"VAX_TYPE\"] == \"COVID19\"\n",
|
|
"\n",
|
|
" def _isFlu(self, dataFrame):\n",
|
|
" return dataFrame[\"VAX_TYPE\"].str.startswith(\"FLU\")\n",
|
|
"\n",
|
|
" def _isManufacturer(self, dataFrame, manufacturer):\n",
|
|
" return dataFrame[\"VAX_MANU\"] == manufacturer if manufacturer is not None else True\n",
|
|
"\n",
|
|
" def _isDose(self, dataFrame, dose):\n",
|
|
" return dataFrame[\"VAX_DOSE_SERIES\"].str.contains(dose) if dose is not None else True\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c62cfaff",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class SummationTableFactory:\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createSummationTable(\n",
|
|
" groupBy,\n",
|
|
" columnNameMappingsDict = {\n",
|
|
" \"DIED_size\": \"Total Number of Adverse Reaction Reports\",\n",
|
|
" \"DIED_sum\": \"Deaths\",\n",
|
|
" \"L_THREAT_sum\": \"Life Threatening Illnesses\",\n",
|
|
" \"DISABLE_sum\": \"Disabilities\",\n",
|
|
" 'HOSPITAL_sum': 'Hospitalisations',\n",
|
|
" 'ER_VISIT_sum': 'Emergency Room or Doctor Visits'\n",
|
|
" }):\n",
|
|
"\n",
|
|
" summationTable = groupBy.agg({\n",
|
|
" 'DIED': ['sum', 'size'],\n",
|
|
" 'L_THREAT': 'sum',\n",
|
|
" 'DISABLE': 'sum',\n",
|
|
" 'HOSPITAL': 'sum',\n",
|
|
" 'ER_VISIT': 'sum',\n",
|
|
" 'SEVERE': 'sum'\n",
|
|
" })\n",
|
|
" SummationTableFactory._flattenColumns(summationTable)\n",
|
|
" return summationTable.rename(columns = columnNameMappingsDict)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createSummationTableHavingSevereReportsColumn(dataFrame):\n",
|
|
" summationTable = SummationTableFactory.createSummationTable(\n",
|
|
" dataFrame,\n",
|
|
" columnNameMappingsDict = {\n",
|
|
" \"DIED_size\": \"Total Number of Adverse Reaction Reports\",\n",
|
|
" \"DIED_sum\": \"Deaths\",\n",
|
|
" \"L_THREAT_sum\": \"Life Threatening Illnesses\",\n",
|
|
" \"DISABLE_sum\": \"Disabilities\",\n",
|
|
" \"SEVERE_sum\": \"Severities\"\n",
|
|
" })\n",
|
|
" summationTable['Severe reports (%)'] = summationTable['Severities'] / summationTable['Total Number of Adverse Reaction Reports'] * 100\n",
|
|
" summationTable = summationTable[['Total Number of Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Severe reports (%)']]\n",
|
|
" return summationTable\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _flattenColumns(dataFrame):\n",
|
|
" dataFrame.columns = [\"_\".join(a) for a in dataFrame.columns.to_flat_index()]\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "99945ca8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"class BatchCodeTableFactory:\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createBatchCodeTable(dataFrame : pd.DataFrame, dose, minADRsForLethality = None):\n",
|
|
" dataFrame = DataFrameFilter().filterByCovid19(dataFrame)\n",
|
|
" dataFrame = DataFrameFilter().filterBy(dataFrame, dose = dose)\n",
|
|
" batchCodeTable = BatchCodeTableFactory._createSummationTableByVAX_LOT(dataFrame)[\n",
|
|
" [\n",
|
|
" 'Total Number of Adverse Reaction Reports',\n",
|
|
" 'Deaths',\n",
|
|
" 'Disabilities',\n",
|
|
" 'Life Threatening Illnesses',\n",
|
|
" 'Company',\n",
|
|
" 'Lethality'\n",
|
|
" ]]\n",
|
|
" if minADRsForLethality is not None:\n",
|
|
" batchCodeTable.loc[batchCodeTable['Total Number of Adverse Reaction Reports'] < minADRsForLethality, 'Lethality'] = np.nan\n",
|
|
" return batchCodeTable\n",
|
|
"\n",
|
|
" # create table from https://www.howbadismybatch.com/combined.html\n",
|
|
" @staticmethod\n",
|
|
" def createSevereEffectsBatchCodeTable(dataFrame : pd.DataFrame, dose):\n",
|
|
" dataFrame = DataFrameFilter().filterByCovid19(dataFrame)\n",
|
|
" dataFrame = DataFrameFilter().filterBy(dataFrame, dose = dose)\n",
|
|
" return BatchCodeTableFactory._createSummationTableByVAX_LOT(dataFrame)[\n",
|
|
" [\n",
|
|
" 'Total Number of Adverse Reaction Reports', \n",
|
|
" 'Deaths',\n",
|
|
" 'Disabilities',\n",
|
|
" 'Life Threatening Illnesses',\n",
|
|
" 'Hospitalisations',\n",
|
|
" 'Emergency Room or Doctor Visits',\n",
|
|
" 'Company'\n",
|
|
" ]]\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _createSummationTableByVAX_LOT(dataFrame):\n",
|
|
" batchCodeTable = SummationTableFactory.createSummationTable(dataFrame.groupby('VAX_LOT'))\n",
|
|
" batchCodeTable['Lethality'] = batchCodeTable['Deaths'] / batchCodeTable['Total Number of Adverse Reaction Reports'] * 100\n",
|
|
" batchCodeTable = batchCodeTable[\n",
|
|
" [\n",
|
|
" 'Total Number of Adverse Reaction Reports',\n",
|
|
" 'Deaths',\n",
|
|
" 'Disabilities',\n",
|
|
" 'Life Threatening Illnesses',\n",
|
|
" 'Hospitalisations',\n",
|
|
" 'Emergency Room or Doctor Visits',\n",
|
|
" 'Lethality'\n",
|
|
" ]]\n",
|
|
" batchCodeTable = batchCodeTable.sort_values(by = 'Total Number of Adverse Reaction Reports', ascending = False)\n",
|
|
" return BatchCodeTableFactory._addCompanyColumn(batchCodeTable, BatchCodeTableFactory._createCompanyByBatchCodeTable(dataFrame))\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _addCompanyColumn(batchCodeTable, companyByBatchCodeTable):\n",
|
|
" return pd.merge(\n",
|
|
" batchCodeTable,\n",
|
|
" companyByBatchCodeTable,\n",
|
|
" how = 'left',\n",
|
|
" left_index = True,\n",
|
|
" right_index = True,\n",
|
|
" validate = 'one_to_one')\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _createCompanyByBatchCodeTable(dataFrame):\n",
|
|
" return BatchCodeTableFactory._createManufacturerByBatchCodeTable(dataFrame).rename(columns = {\"VAX_MANU\": \"Company\"})\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _createManufacturerByBatchCodeTable(dataFrame):\n",
|
|
" manufacturerByBatchCodeTable = dataFrame[['VAX_LOT', 'VAX_MANU']]\n",
|
|
" manufacturerByBatchCodeTable = manufacturerByBatchCodeTable.drop_duplicates(subset = ['VAX_LOT'])\n",
|
|
" return manufacturerByBatchCodeTable.set_index('VAX_LOT')\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "41d4fa30",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class DoseTableFactory:\n",
|
|
" \n",
|
|
" @staticmethod\n",
|
|
" def createDoseTable(dataFrame):\n",
|
|
" dataFrame = DataFrameFilter().filterByCovid19(dataFrame)\n",
|
|
" return SummationTableFactory.createSummationTableHavingSevereReportsColumn(\n",
|
|
" dataFrame.groupby(\n",
|
|
" dataFrame['VAX_DOSE_SERIES'].rename('Dose')))\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createDoseByMonthTable(dataFrame):\n",
|
|
" dataFrame = DataFrameFilter().filterByCovid19(dataFrame)\n",
|
|
" return SummationTableFactory.createSummationTableHavingSevereReportsColumn(\n",
|
|
" dataFrame.groupby(\n",
|
|
" [\n",
|
|
" dataFrame['RECVDATE'].dt.year.rename('Year'),\n",
|
|
" dataFrame['RECVDATE'].dt.month.rename('Month'),\n",
|
|
" dataFrame['VAX_DOSE_SERIES'].rename('Dose')\n",
|
|
" ]))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c40bd0f0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pycountry\n",
|
|
"\n",
|
|
"class CountryColumnAdder:\n",
|
|
" \n",
|
|
" @staticmethod\n",
|
|
" def addCountryColumn(dataFrame, countryColumnName):\n",
|
|
" dataFrame[countryColumnName] = dataFrame.apply(\n",
|
|
" lambda row:\n",
|
|
" CountryColumnAdder._getCountryNameOfSplttypeOrDefault(\n",
|
|
" splttype = row['SPLTTYPE'],\n",
|
|
" default = 'Unknown Country'),\n",
|
|
" axis = 'columns')\n",
|
|
" return dataFrame.astype({countryColumnName: \"string\"})\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _getCountryNameOfSplttypeOrDefault(splttype, default):\n",
|
|
" if not isinstance(splttype, str):\n",
|
|
" return default\n",
|
|
" \n",
|
|
" country = pycountry.countries.get(alpha_2 = splttype[:2])\n",
|
|
" return country.name if country is not None else default"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3abe3384",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pycountry\n",
|
|
"\n",
|
|
"class SevereColumnAdder:\n",
|
|
" \n",
|
|
" @staticmethod\n",
|
|
" def addSevereColumn(dataFrame):\n",
|
|
" dataFrame['SEVERE'] = (dataFrame['DIED'] + dataFrame['L_THREAT'] + dataFrame['DISABLE']) > 0\n",
|
|
" dataFrame['SEVERE'].replace({True: 1, False: 0}, inplace = True)\n",
|
|
" return dataFrame\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "09e6b511",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class InternationalLotTableFactory:\n",
|
|
" \n",
|
|
" @staticmethod\n",
|
|
" def createInternationalLotTable(dataFrame):\n",
|
|
" dataFrame = DataFrameFilter().filterByCovid19(dataFrame)\n",
|
|
" internationalLotTable = InternationalLotTableFactory._createInternationalLotTable(dataFrame)\n",
|
|
" return internationalLotTable.sort_values(by = 'Severe reports (%)', ascending = False)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createBatchCodeTableByCountry(dataFrame : pd.DataFrame, country):\n",
|
|
" dataFrame = DataFrameFilter().filterByCovid19(dataFrame)\n",
|
|
" batchCodeTable = InternationalLotTableFactory._createBatchCodeTableByCountry(dataFrame, country)\n",
|
|
" return batchCodeTable.sort_values(by = 'Severe reports (%)', ascending = False)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _createInternationalLotTable(dataFrame):\n",
|
|
" countryColumnName = 'Country'\n",
|
|
" dataFrame = CountryColumnAdder.addCountryColumn(dataFrame, countryColumnName = countryColumnName)\n",
|
|
" return SummationTableFactory.createSummationTableHavingSevereReportsColumn(dataFrame.groupby(dataFrame[countryColumnName]))\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def _createBatchCodeTableByCountry(dataFrame : pd.DataFrame, country):\n",
|
|
" countryColumnName = 'Country'\n",
|
|
" dataFrame = CountryColumnAdder.addCountryColumn(dataFrame, countryColumnName = countryColumnName)\n",
|
|
" dataFrame = DataFrameFilter().filterByCountry(dataFrame, country = country, countryColumnName = countryColumnName)\n",
|
|
" return SummationTableFactory.createSummationTableHavingSevereReportsColumn(dataFrame.groupby('VAX_LOT'))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6aa28541",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"\n",
|
|
"class IOUtils:\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def saveDataFrame(dataFrame, file):\n",
|
|
" IOUtils.saveDataFrameAsExcelFile(dataFrame, file)\n",
|
|
" IOUtils.saveDataFrameAsHtml(dataFrame, file)\n",
|
|
" IOUtils.saveDataFrameAsJson(dataFrame, file)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def saveDataFrameAsExcelFile(dataFrame, file):\n",
|
|
" IOUtils.ensurePath(file)\n",
|
|
" dataFrame.to_excel(file + '.xlsx')\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def saveDataFrameAsHtml(dataFrame, file):\n",
|
|
" IOUtils.ensurePath(file)\n",
|
|
" dataFrame.reset_index().to_html(\n",
|
|
" file + '.html',\n",
|
|
" index = False,\n",
|
|
" table_id = 'batchCodeTable',\n",
|
|
" classes = 'display',\n",
|
|
" justify = 'unset',\n",
|
|
" border = 0)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def saveDataFrameAsJson(dataFrame, file):\n",
|
|
" IOUtils.ensurePath(file)\n",
|
|
" dataFrame.reset_index().to_json(\n",
|
|
" file + '.json',\n",
|
|
" orient = \"split\",\n",
|
|
" index = False)\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def ensurePath(file):\n",
|
|
" directory = os.path.dirname(file)\n",
|
|
" if not os.path.exists(directory):\n",
|
|
" os.makedirs(directory)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3dacedfd",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import unittest"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "fcc855dd",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class TestHelper:\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def createDataFrame(index, columns, data, dtypes = {}):\n",
|
|
" return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ccb9838d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class DataFrameNormalizerTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_convertVAX_LOTColumnToUpperCase(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_LOT'],\n",
|
|
" data = [ ['037K20A'],\n",
|
|
" ['025l20A'],\n",
|
|
" ['025L20A']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\",\n",
|
|
" \"1996874\"])\n",
|
|
" \n",
|
|
" # When\n",
|
|
" DataFrameNormalizer.convertVAX_LOTColumnToUpperCase(dataFrame)\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_LOT'],\n",
|
|
" data = [ ['037K20A'],\n",
|
|
" ['025L20A'],\n",
|
|
" ['025L20A']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\",\n",
|
|
" \"1996874\"])\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e59a1825",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class DataFrameFilterTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_filterByFlu(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [0, 0, 0, 'FLU(H1N1)', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLU3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLU4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUA3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUA4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUC3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUC4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUN(H1N1)', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUN3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUN4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUR3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUR4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUX', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUX(H1N1)', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 1, 'COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"801410\",\n",
|
|
" \"801411\",\n",
|
|
" \"801412\",\n",
|
|
" \"801413\",\n",
|
|
" \"801414\",\n",
|
|
" \"801415\",\n",
|
|
" \"801416\",\n",
|
|
" \"801417\",\n",
|
|
" \"801418\",\n",
|
|
" \"801419\",\n",
|
|
" \"801420\",\n",
|
|
" \"801421\",\n",
|
|
" \"801422\",\n",
|
|
" \"801423\",\n",
|
|
" \"801424\"])\n",
|
|
" dataFrameFilter = DataFrameFilter()\n",
|
|
" \n",
|
|
" # When\n",
|
|
" dataFrameActual = dataFrameFilter.filterByFlu(dataFrame)\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [0, 0, 0, 'FLU(H1N1)', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLU3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLU4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUA3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUA4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUC3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUC4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUN(H1N1)', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUN3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUN4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUR3', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUR4', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUX', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1'],\n",
|
|
" [0, 0, 0, 'FLUX(H1N1)', 'GLAXOSMITHKLINE BIOLOGICALS', '5R3J5', '1']],\n",
|
|
" index = [\n",
|
|
" \"801410\",\n",
|
|
" \"801411\",\n",
|
|
" \"801412\",\n",
|
|
" \"801413\",\n",
|
|
" \"801414\",\n",
|
|
" \"801415\",\n",
|
|
" \"801416\",\n",
|
|
" \"801417\",\n",
|
|
" \"801418\",\n",
|
|
" \"801419\",\n",
|
|
" \"801420\",\n",
|
|
" \"801421\",\n",
|
|
" \"801422\",\n",
|
|
" \"801423\"])\n",
|
|
" assert_frame_equal(dataFrameActual, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_filterByCovid19_filterBy(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [1, 0, 0],\n",
|
|
" [0, 0, 1]],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" },\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [0, 0, 0],\n",
|
|
" [0, 0, 1]],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ])\n",
|
|
" dataFrameFilter = DataFrameFilter()\n",
|
|
" \n",
|
|
" # When\n",
|
|
" dataFrame = dataFrameFilter.filterByCovid19(dataFrame)\n",
|
|
" dataFrame = dataFrameFilter.filterBy(dataFrame, manufacturer = \"MODERNA\", dose = '1')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" [0, 0, 1, 'COVID19', 'MODERNA', '025L20A', '1'],\n",
|
|
" [0, 0, 1, 'COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_filterByDose(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 1, 0, 1, 1],\n",
|
|
" [0, 0, 1, 0, 1]],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'PFIZER\\BIONTECH', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ])\n",
|
|
" dataFrameFilter = DataFrameFilter()\n",
|
|
" dataFrame = dataFrameFilter.filterByCovid19(dataFrame)\n",
|
|
"\n",
|
|
" # When\n",
|
|
" dataFrame = dataFrameFilter.filterBy(dataFrame, dose = '1')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [1, 1, 0, 1, 1, 'COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" [0, 0, 1, 0, 1, 'COVID19', 'PFIZER\\BIONTECH', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_filterByFirstDose(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1048786\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n",
|
|
" ['COVID19', 'MODERNA', '030L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ])\n",
|
|
" dataFrameFilter = DataFrameFilter()\n",
|
|
" \n",
|
|
" # When\n",
|
|
" dataFrame = dataFrameFilter.filterByCovid19(dataFrame)\n",
|
|
" dataFrame = dataFrameFilter.filterBy(dataFrame, manufacturer = \"MODERNA\", dose = '1')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_filterBySecondDose(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE'],\n",
|
|
" data = [ [1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1048786\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '016M20A', '2'],\n",
|
|
" ['COVID19', 'MODERNA', '030L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ])\n",
|
|
" dataFrameFilter = DataFrameFilter()\n",
|
|
"\n",
|
|
" # When\n",
|
|
" dataFrame = dataFrameFilter.filterByCovid19(dataFrame)\n",
|
|
" dataFrame = dataFrameFilter.filterBy(dataFrame, manufacturer = \"MODERNA\", dose = '2')\n",
|
|
" \n",
|
|
" # Then\n",
|
|
" dataFrameExpected = TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '016M20A', '2']],\n",
|
|
" index = [\n",
|
|
" \"1048786\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e14465d7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class BatchCodeTableFactoryTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_createSevereEffectsBatchCodeTable(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 1, 0, 1, 1],\n",
|
|
" [0, 0, 1, 0, 1]],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'PFIZER\\BIONTECH', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ])\n",
|
|
" dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n",
|
|
"\n",
|
|
" # When\n",
|
|
" batchCodeTable = BatchCodeTableFactory.createSevereEffectsBatchCodeTable(dataFrame, '1')\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" batchCodeTableExpected = pd.DataFrame(\n",
|
|
" data = {\n",
|
|
" 'Total Number of Adverse Reaction Reports': [1, 1],\n",
|
|
" 'Deaths': [0, 1],\n",
|
|
" 'Disabilities': [1, 0],\n",
|
|
" 'Life Threatening Illnesses': [0, 1],\n",
|
|
" 'Hospitalisations': [0, 1],\n",
|
|
" 'Emergency Room or Doctor Visits': [1, 1],\n",
|
|
" 'Company': ['PFIZER\\BIONTECH', 'MODERNA']\n",
|
|
" },\n",
|
|
" index = pd.Index(['025L20A', '037K20A'], name = 'VAX_LOT'))\n",
|
|
" assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_createBatchCodeTable(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 0, 0, 0, 0],\n",
|
|
" [0, 0, 1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" },\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ])\n",
|
|
" self._test_createBatchCodeTable(dataFrame, '1')\n",
|
|
"\n",
|
|
" def test_createBatchCodeTable_minADRsForLethality(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" [\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 0, 0, 0, 0],\n",
|
|
" [0, 0, 1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"0916600\",\n",
|
|
" \"0916601\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" },\n",
|
|
" {\n",
|
|
" 'VAERSDATA': TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 1, 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"]),\n",
|
|
" 'VAERSVAX': TestHelper.createDataFrame(\n",
|
|
" columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n",
|
|
" data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n",
|
|
" ['COVID19', 'MODERNA', '025L20A', '1']],\n",
|
|
" index = [\n",
|
|
" \"1996873\",\n",
|
|
" \"1996874\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" }\n",
|
|
" ])\n",
|
|
" dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n",
|
|
"\n",
|
|
" # When\n",
|
|
" batchCodeTable = BatchCodeTableFactory.createBatchCodeTable(dataFrame, dose = '1', minADRsForLethality = 2)\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" batchCodeTableExpected = pd.DataFrame(\n",
|
|
" data = {\n",
|
|
" 'Total Number of Adverse Reaction Reports': [2, 1],\n",
|
|
" 'Deaths': [0, 1],\n",
|
|
" 'Disabilities': [2, 0],\n",
|
|
" 'Life Threatening Illnesses': [0, 0],\n",
|
|
" 'Company': ['MODERNA', 'MODERNA'],\n",
|
|
" 'Lethality': [0/2 * 100, np.nan]\n",
|
|
" },\n",
|
|
" index = pd.Index(['025L20A', '037K20A'], name = 'VAX_LOT'))\n",
|
|
" assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n",
|
|
"\n",
|
|
" def test_createBatchCodeTableFromFiles(self):\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n",
|
|
" VaersDescrReader(dataDir = \"test/VAERS\").readVaersDescrs([\"2021\", \"2022\"]))\n",
|
|
" DataFrameNormalizer.normalize(dataFrame)\n",
|
|
" self._test_createBatchCodeTable(dataFrame, '1')\n",
|
|
"\n",
|
|
" def _test_createBatchCodeTable(self, dataFrame, dose):\n",
|
|
" dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n",
|
|
"\n",
|
|
" # When\n",
|
|
" batchCodeTable = BatchCodeTableFactory.createBatchCodeTable(dataFrame, dose)\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" batchCodeTableExpected = pd.DataFrame(\n",
|
|
" data = {\n",
|
|
" 'Total Number of Adverse Reaction Reports': [2, 1],\n",
|
|
" 'Deaths': [0, 1],\n",
|
|
" 'Disabilities': [2, 0],\n",
|
|
" 'Life Threatening Illnesses': [0, 0],\n",
|
|
" 'Company': ['MODERNA', 'MODERNA'],\n",
|
|
" 'Lethality': [0/2 * 100, 1/1 * 100]\n",
|
|
" },\n",
|
|
" index = pd.Index(['025L20A', '037K20A'], name = 'VAX_LOT'))\n",
|
|
" assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "44c121ec",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class DoseTableFactoryTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_createDoseTable(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '016M20A', '2', 0, 0],\n",
|
|
" [1, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1', 0, 0],\n",
|
|
" [1, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\",\n",
|
|
" \"4711\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n",
|
|
" \n",
|
|
" # When\n",
|
|
" doseTable = DoseTableFactory.createDoseTable(dataFrame)\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" assert_frame_equal(\n",
|
|
" doseTable,\n",
|
|
" pd.DataFrame(\n",
|
|
" data = {\n",
|
|
" 'Total Number of Adverse Reaction Reports': [2, 1],\n",
|
|
" 'Deaths': [2, 1],\n",
|
|
" 'Disabilities': [1, 0],\n",
|
|
" 'Life Threatening Illnesses': [1, 0],\n",
|
|
" 'Severe reports (%)': [2/2 * 100, 1/1 * 100]\n",
|
|
" },\n",
|
|
" index = pd.Index(['1', '2'], dtype = \"string\", name = 'Dose')))\n",
|
|
" \n",
|
|
" def test_createDoseByMonthTable(self):\n",
|
|
" # Given\n",
|
|
" parseDate = lambda dateStr: pd.to_datetime(dateStr, format = \"%m/%d/%Y\")\n",
|
|
" dataFrame = TestHelper.createDataFrame(\n",
|
|
" columns = ['RECVDATE', 'DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [parseDate('01/01/2021'), 1, 0, 0, 'COVID19', 'MODERNA', '016M20A', '2', 0, 0],\n",
|
|
" [parseDate('01/01/2021'), 1, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1', 0, 0],\n",
|
|
" [parseDate('01/01/2021'), 1, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\",\n",
|
|
" \"4711\"],\n",
|
|
" dtypes = {'VAX_DOSE_SERIES': \"string\"})\n",
|
|
" dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n",
|
|
" \n",
|
|
" # When\n",
|
|
" doseByMonthTable = DoseTableFactory.createDoseByMonthTable(dataFrame)\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" assert_frame_equal(\n",
|
|
" doseByMonthTable,\n",
|
|
" pd.DataFrame(\n",
|
|
" data = {\n",
|
|
" 'Total Number of Adverse Reaction Reports': [2, 1],\n",
|
|
" 'Deaths': [2, 1],\n",
|
|
" 'Disabilities': [1, 0],\n",
|
|
" 'Life Threatening Illnesses': [1, 0],\n",
|
|
" 'Severe reports (%)': [2/2 * 100, 1/1 * 100]\n",
|
|
" },\n",
|
|
" index = pd.MultiIndex.from_tuples(\n",
|
|
" [\n",
|
|
" (2021, 1, '1'),\n",
|
|
" (2021, 1, '2'),\n",
|
|
" ],\n",
|
|
" names = ('Year', 'Month', 'Dose'))),\n",
|
|
" check_index_type = False)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c784bfef",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pandas.testing import assert_frame_equal\n",
|
|
"\n",
|
|
"class InternationalLotTableFactoryTest(unittest.TestCase):\n",
|
|
"\n",
|
|
" def test_createInternationalLotTable(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES', 'SPLTTYPE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '016M20A', '2', 'GBPFIZER INC2020486806', 0, 0],\n",
|
|
" [1, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1', 'FRMODERNATX, INC.MOD20224', 0, 0],\n",
|
|
" [1, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0],\n",
|
|
" [0, 0, 0, 'COVID19', 'MODERNA', '030L20B', '1', 'dummy'],\n",
|
|
" [0, 0, 0, 'COVID19', 'MODERNA', '030L20B', '1', 123]],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\",\n",
|
|
" \"4711\",\n",
|
|
" \"0815\",\n",
|
|
" \"0816\"])\n",
|
|
" dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n",
|
|
" \n",
|
|
" # When\n",
|
|
" internationalLotTable = InternationalLotTableFactory.createInternationalLotTable(dataFrame)\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" assert_frame_equal(\n",
|
|
" internationalLotTable,\n",
|
|
" TestHelper.createDataFrame(\n",
|
|
" columns = ['Total Number of Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Severe reports (%)'],\n",
|
|
" data = [ [2, 2, 1, 1, 2/2 * 100],\n",
|
|
" [1, 1, 0, 0, 1/1 * 100],\n",
|
|
" [2, 0, 0, 0, 0/2 * 100]],\n",
|
|
" index = pd.Index(\n",
|
|
" [\n",
|
|
" 'France',\n",
|
|
" 'United Kingdom',\n",
|
|
" 'Unknown Country'\n",
|
|
" ],\n",
|
|
" dtype = \"string\",\n",
|
|
" name = 'Country')))\n",
|
|
"\n",
|
|
" def test_createBatchCodeTableByCountry(self):\n",
|
|
" # Given\n",
|
|
" dataFrame = TestHelper.createDataFrame(\n",
|
|
" columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES', 'SPLTTYPE', 'HOSPITAL', 'ER_VISIT'],\n",
|
|
" data = [ [1, 0, 0, 'COVID19', 'MODERNA', '016M20A', '2', 'GBPFIZER INC2020486806', 0, 0],\n",
|
|
" [0, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1', 'FRMODERNATX, INC.MOD20224', 0, 0],\n",
|
|
" [1, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0],\n",
|
|
" [0, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0]],\n",
|
|
" index = [\n",
|
|
" \"1048786\",\n",
|
|
" \"1048786\",\n",
|
|
" \"4711\",\n",
|
|
" \"0815\"])\n",
|
|
" dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n",
|
|
" \n",
|
|
" # When\n",
|
|
" batchCodeTable = InternationalLotTableFactory.createBatchCodeTableByCountry(dataFrame, 'France')\n",
|
|
"\n",
|
|
" # Then\n",
|
|
" assert_frame_equal(\n",
|
|
" batchCodeTable,\n",
|
|
" TestHelper.createDataFrame(\n",
|
|
" columns = ['Total Number of Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Severe reports (%)'],\n",
|
|
" data = [ [2, 1, 2, 2, 2/2 * 100],\n",
|
|
" [1, 0, 0, 0, 0/1 * 100]],\n",
|
|
" index = pd.Index(\n",
|
|
" [\n",
|
|
" '030L20B',\n",
|
|
" '030L20A'\n",
|
|
" ],\n",
|
|
" name = 'VAX_LOT')),\n",
|
|
" check_dtype = False)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5a8bff1b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"unittest.main(argv = [''], verbosity = 2, exit = False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "86e0e4f2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def getVaers(vaersDescrsReaderFunc):\n",
|
|
" vaersDescrs = vaersDescrsReaderFunc()\n",
|
|
" dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(vaersDescrs)\n",
|
|
" DataFrameNormalizer.normalize(dataFrame)\n",
|
|
" dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n",
|
|
" return dataFrame\n",
|
|
"\n",
|
|
"def getVaersDescrReader():\n",
|
|
" return VaersDescrReader(dataDir = \"VAERS\")\n",
|
|
"\n",
|
|
"def getAllVaers():\n",
|
|
" return getVaers(lambda: getVaersDescrReader().readVaersDescrs([\"2021\", \"2022\"]))\n",
|
|
"\n",
|
|
"def getNonDomesticVaers():\n",
|
|
" return getVaers(lambda: [getVaersDescrReader().readNonDomesticVaersDescr()])\n",
|
|
"\n",
|
|
"def getInternationalVaers():\n",
|
|
" return pd.concat([getAllVaers(), getNonDomesticVaers()])\n",
|
|
"\n",
|
|
"def getVaersForYear(year):\n",
|
|
" return getVaers(lambda: [getVaersDescrReader().readVaersDescr(year)])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9ee014eb",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"vaers = getAllVaers()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c96391fc",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"nonDomesticVaers = getNonDomesticVaers()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "404b496a",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"internationalVaers = getInternationalVaers()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "987a04d1",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Batch codes"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7edf87d7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def saveBatchCodeTable(vaers, file):\n",
|
|
" batchCodeTable = BatchCodeTableFactory.createBatchCodeTable(vaers, dose = '1', minADRsForLethality = 100)\n",
|
|
" batchCodeTable.index.set_names(\"Batch\", inplace = True)\n",
|
|
" display(batchCodeTable)\n",
|
|
" IOUtils.saveDataFrame(batchCodeTable, file)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ab170c16",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# https://www.howbadismybatch.com/moderna.html\n",
|
|
"saveBatchCodeTable(internationalVaers, \"../data/batchCodeTable\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f677b620",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Short-list of 2000 batches having severe effects"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "bc56831d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def saveSevereEffectsBatchCodeTable(vaers, file):\n",
|
|
" severeEffectsBatchCodeTable = BatchCodeTableFactory.createSevereEffectsBatchCodeTable(vaers, dose = '1')\n",
|
|
" display(severeEffectsBatchCodeTable)\n",
|
|
" IOUtils.saveDataFrame(severeEffectsBatchCodeTable, file)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ace3fed9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"saveSevereEffectsBatchCodeTable(vaers, 'results/severeEffects')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "1b228a16",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Variation in Effect of First and Second Doses"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "202f7c3f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# https://www.howbadismybatch.com/firstsecond.html\n",
|
|
"DoseTableFactory.createDoseTable(vaers)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b333e5fb",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"doseByMonthTable = DoseTableFactory.createDoseByMonthTable(vaers)\n",
|
|
"IOUtils.saveDataFrame(doseByMonthTable, 'results/firstsecond/doseByMonthTable')\n",
|
|
"doseByMonthTable"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "075aa6c9",
|
|
"metadata": {},
|
|
"source": [
|
|
"### International Deadly Lots"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8f8880f4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# https://www.howbadismybatch.com/international.html"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "54e03231",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"internationalLotTable = InternationalLotTableFactory.createInternationalLotTable(nonDomesticVaers)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7e80e958",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"internationalLotTable = internationalLotTable[internationalLotTable['Total Number of Adverse Reaction Reports'] > 50]\n",
|
|
"IOUtils.saveDataFrame(internationalLotTable, 'results/international/International_Deadly_Lots')\n",
|
|
"internationalLotTable"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ff259a35",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def createAndSaveAndDisplayBatchCodeTableByCountry(nonDomesticVaers, country):\n",
|
|
" batchCodeTable = InternationalLotTableFactory.createBatchCodeTableByCountry(nonDomesticVaers, country)\n",
|
|
" batchCodeTable = batchCodeTable[batchCodeTable['Total Number of Adverse Reaction Reports'] > 50]\n",
|
|
" IOUtils.saveDataFrame(batchCodeTable, 'results/international/' + country)\n",
|
|
" display(country + \":\", batchCodeTable)\n",
|
|
"\n",
|
|
"def createAndSaveAndDisplayBatchCodeTablesByCountry(nonDomesticVaers, countries):\n",
|
|
" for country in countries:\n",
|
|
" createAndSaveAndDisplayBatchCodeTableByCountry(nonDomesticVaers, country)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7e7e01a5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"createAndSaveAndDisplayBatchCodeTablesByCountry(\n",
|
|
" nonDomesticVaers,\n",
|
|
" [\n",
|
|
" 'United Kingdom',\n",
|
|
" 'France',\n",
|
|
" 'Germany',\n",
|
|
" 'Japan',\n",
|
|
" 'Italy',\n",
|
|
" 'Austria',\n",
|
|
" 'Netherlands',\n",
|
|
" 'Spain',\n",
|
|
" 'Belgium',\n",
|
|
" 'Sweden',\n",
|
|
" 'Portugal',\n",
|
|
" 'Australia'\n",
|
|
" ])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ba02139d",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Batch Clusters"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9649a32d",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### Pfizer Batches"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f6e460ab",
|
|
"metadata": {},
|
|
"source": [
|
|
"see https://www.howbadismybatch.com/clusters.html"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b769466d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def createADRsByVAX_LOTTable(vaers, manufacturer):\n",
|
|
" dataFrame = DataFrameFilter().filterByCovid19(vaers)\n",
|
|
" dataFrame = DataFrameFilter().filterBy(dataFrame, manufacturer = manufacturer)\n",
|
|
" batchCodeTable = BatchCodeTableFactory._createSummationTableByVAX_LOT(dataFrame)[['Total Number of Adverse Reaction Reports']].reset_index()\n",
|
|
" return batchCodeTable\n",
|
|
"\n",
|
|
"def filterColumnOfDataFrameWithRegexp(dataFrame, column, regexp):\n",
|
|
" return dataFrame[dataFrame[column].apply(lambda columnValue: bool(regexp.match(columnValue)))]\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "020b0d90",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import re\n",
|
|
"\n",
|
|
"batchCodeTable = createADRsByVAX_LOTTable(vaers, \"PFIZER\\BIONTECH\")\n",
|
|
"batchCodeTable['VAX_LOT_PREFIX'] = batchCodeTable['VAX_LOT'].str[:2]\n",
|
|
"batchCodeTable = batchCodeTable.sort_values(by = 'VAX_LOT_PREFIX', ascending = True)\n",
|
|
"twoLetterPrefix = re.compile(r'^[a-zA-Z]{2}')\n",
|
|
"batchCodeTable = filterColumnOfDataFrameWithRegexp(dataFrame = batchCodeTable, column = 'VAX_LOT_PREFIX', regexp = twoLetterPrefix)\n",
|
|
"batchCodeTable = batchCodeTable[batchCodeTable['VAX_LOT_PREFIX'].isin(['EN', 'EP', 'ER', 'EW', 'FA', 'FC', 'FD', 'FE', 'FH'])]\n",
|
|
"batchCodeTable = batchCodeTable[batchCodeTable['Total Number of Adverse Reaction Reports'] > 400]\n",
|
|
"batchCodeTable"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "02201726",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import seaborn as sns\n",
|
|
"\n",
|
|
"sns.set(rc = {'figure.figsize': (11.7, 8.27)})\n",
|
|
"sns.set_theme()\n",
|
|
"chart = sns.stripplot(x = \"VAX_LOT_PREFIX\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d6000b48",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sns.pointplot(x = \"VAX_LOT_PREFIX\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable, estimator = np.mean)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cf53c8c8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import seaborn as sns\n",
|
|
"sns.set_theme(style = \"ticks\", palette = \"pastel\")\n",
|
|
"\n",
|
|
"sns.boxplot(x = \"VAX_LOT_PREFIX\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "731c27a5",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### Moderna Batches"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b4a9c489",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import re\n",
|
|
"\n",
|
|
"batchCodeTable = createADRsByVAX_LOTTable(vaers, \"MODERNA\")\n",
|
|
"modernaBatchCodePrefix = re.compile(r'^[0-9]{3}[a-zA-Z]')\n",
|
|
"batchCodeTable = filterColumnOfDataFrameWithRegexp(dataFrame = batchCodeTable, column = 'VAX_LOT', regexp = modernaBatchCodePrefix)\n",
|
|
"batchCodeTable['CONCENTRATION'] = batchCodeTable['VAX_LOT'].str[3]\n",
|
|
"batchCodeTable = batchCodeTable.sort_values(by = 'CONCENTRATION', ascending = True)\n",
|
|
"batchCodeTable = batchCodeTable[batchCodeTable['Total Number of Adverse Reaction Reports'] > 400]\n",
|
|
"batchCodeTable"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e26c9d85",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import seaborn as sns\n",
|
|
"\n",
|
|
"order = ['J', 'K', 'L', 'M', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H']\n",
|
|
"\n",
|
|
"sns.set(rc = {'figure.figsize': (11.7, 8.27)})\n",
|
|
"sns.set_theme()\n",
|
|
"chart = sns.stripplot(x = \"CONCENTRATION\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable, order = order)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d1de13c7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sns.pointplot(x = \"CONCENTRATION\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable, estimator = np.mean, order = order)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "29ae8ca2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import seaborn as sns\n",
|
|
"sns.set_theme(style = \"ticks\", palette = \"pastel\")\n",
|
|
"\n",
|
|
"sns.boxplot(x = \"CONCENTRATION\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable, order = order)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "259b6474",
|
|
"metadata": {},
|
|
"source": [
|
|
"### COVID-19 Vaccines vs. Flu Vaccines"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "fa5c8480",
|
|
"metadata": {},
|
|
"source": [
|
|
"see https://www.bitchute.com/video/4HlIyBmOEJeY/ and https://www.bitchute.com/video/8wJYP2NpGwN2/"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "37ccbee6",
|
|
"metadata": {},
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|