{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "9de5907f-18f5-4cb1-903e-26028ff1fa03", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "\n", "pd.set_option('display.max_rows', 100)\n", "pd.set_option('display.max_columns', None)" ] }, { "cell_type": "code", "execution_count": null, "id": "a271254b", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "class VaersDescrReader:\n", " \n", " def __init__(self, dataDir):\n", " self.dataDir = dataDir \n", "\n", " def readVaersDescrsForYears(self, years):\n", " return [self.readVaersDescrForYear(year) for year in years]\n", "\n", " def readVaersDescrForYear(self, year):\n", " return {\n", " 'VAERSDATA': self._readVAERSDATA('{dataDir}/{year}VAERSDATA.csv'.format(dataDir = self.dataDir, year = year)),\n", " 'VAERSVAX': self._readVAERSVAX('{dataDir}/{year}VAERSVAX.csv'.format(dataDir = self.dataDir, year = year))\n", " }\n", "\n", " def readNonDomesticVaersDescr(self):\n", " return {\n", " 'VAERSDATA': self._readVAERSDATA(self.dataDir + \"/NonDomesticVAERSDATA.csv\"),\n", " 'VAERSVAX': self._readVAERSVAX(self.dataDir + \"/NonDomesticVAERSVAX.csv\")\n", " }\n", "\n", " def _readVAERSDATA(self, file):\n", " return self._read_csv(\n", " file = file,\n", " usecols = ['VAERS_ID', 'RECVDATE', 'DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT', 'SPLTTYPE'],\n", " parse_dates = ['RECVDATE'],\n", " date_parser = lambda dateStr: pd.to_datetime(dateStr, format = \"%m/%d/%Y\"))\n", "\n", " def _readVAERSVAX(self, file):\n", " return self._read_csv(\n", " file = file,\n", " usecols = ['VAERS_ID', 'VAX_DOSE_SERIES', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT'],\n", " dtype = {\"VAX_DOSE_SERIES\": \"string\"})\n", "\n", " def _read_csv(self, file, **kwargs):\n", " return pd.read_csv(\n", " file,\n", " index_col = 'VAERS_ID',\n", " encoding = 'latin1',\n", " low_memory = False,\n", " **kwargs)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "7b5d6df0", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "class VaersDescr2DataFrameConverter:\n", "\n", " @staticmethod\n", " def createDataFrameFromDescr(vaersDescr):\n", " return pd.merge(\n", " vaersDescr['VAERSDATA'],\n", " vaersDescr['VAERSVAX'],\n", " how = 'left',\n", " left_index = True,\n", " right_index = True,\n", " validate = 'one_to_many')\n", "\n", " @staticmethod\n", " def createDataFrameFromDescrs(vaersDescrs):\n", " dataFrames = [VaersDescr2DataFrameConverter.createDataFrameFromDescr(vaersDescr) for vaersDescr in vaersDescrs]\n", " return pd.concat(dataFrames)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "6b639196", "metadata": {}, "outputs": [], "source": [ "class DataFrameNormalizer:\n", " \n", " @staticmethod\n", " def normalize(dataFrame):\n", " DataFrameNormalizer.convertVAX_LOTColumnToUpperCase(dataFrame)\n", " DataFrameNormalizer._convertColumnsOfDataFrame_Y_to_1_else_0(\n", " dataFrame,\n", " ['DIED', 'L_THREAT', 'DISABLE', 'HOSPITAL', 'ER_VISIT'])\n", "\n", " @staticmethod\n", " def convertVAX_LOTColumnToUpperCase(dataFrame):\n", " dataFrame['VAX_LOT'] = dataFrame['VAX_LOT'].str.upper()\n", "\n", " @staticmethod\n", " def _convertColumnsOfDataFrame_Y_to_1_else_0(dataFrame, columns):\n", " for column in columns:\n", " DataFrameNormalizer._convertColumnOfDataFrame_Y_to_1_else_0(dataFrame, column)\n", "\n", " @staticmethod\n", " def _convertColumnOfDataFrame_Y_to_1_else_0(dataFrame, column):\n", " dataFrame[column] = DataFrameNormalizer._where(\n", " condition = dataFrame[column] == 'Y',\n", " trueValue = 1,\n", " falseValue = 0)\n", "\n", " @staticmethod\n", " def _where(condition, trueValue, falseValue):\n", " return np.where(condition, trueValue, falseValue) \n", " " ] }, { "cell_type": "code", "execution_count": null, "id": "3ebcba86", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "class DataFrameFilter:\n", " \n", " def filterByCovid19(self, dataFrame):\n", " return dataFrame[self._isCovid19(dataFrame)]\n", "\n", " def _isCovid19(self, dataFrame):\n", " return dataFrame[\"VAX_TYPE\"] == \"COVID19\"\n" ] }, { "cell_type": "code", "execution_count": null, "id": "c62cfaff", "metadata": {}, "outputs": [], "source": [ "class SummationTableFactory:\n", "\n", " @staticmethod\n", " def createSummationTable(\n", " groupBy,\n", " columnNameMappingsDict = {\n", " \"DIED_size\": \"Adverse Reaction Reports\",\n", " \"DIED_sum\": \"Deaths\",\n", " \"L_THREAT_sum\": \"Life Threatening Illnesses\",\n", " \"DISABLE_sum\": \"Disabilities\",\n", " 'HOSPITAL_sum': 'Hospitalisations',\n", " 'ER_VISIT_sum': 'Emergency Room or Doctor Visits'\n", " }):\n", "\n", " summationTable = groupBy.agg({\n", " 'DIED': ['sum', 'size'],\n", " 'L_THREAT': 'sum',\n", " 'DISABLE': 'sum',\n", " 'HOSPITAL': 'sum',\n", " 'ER_VISIT': 'sum',\n", " 'SEVERE': 'sum'\n", " })\n", " SummationTableFactory._flattenColumns(summationTable)\n", " return summationTable.rename(columns = columnNameMappingsDict)\n", "\n", " @staticmethod\n", " def createSummationTableHavingSevereReportsColumn(dataFrame):\n", " summationTable = SummationTableFactory.createSummationTable(\n", " dataFrame,\n", " columnNameMappingsDict = {\n", " \"DIED_size\": \"Adverse Reaction Reports\",\n", " \"DIED_sum\": \"Deaths\",\n", " \"L_THREAT_sum\": \"Life Threatening Illnesses\",\n", " \"DISABLE_sum\": \"Disabilities\",\n", " \"SEVERE_sum\": \"Severities\"\n", " })\n", " summationTable['Severe reports'] = summationTable['Severities'] / summationTable['Adverse Reaction Reports'] * 100\n", " summationTable['Lethality'] = summationTable['Deaths'] / summationTable['Adverse Reaction Reports'] * 100\n", " summationTable = summationTable[\n", " [\n", " 'Adverse Reaction Reports',\n", " 'Deaths',\n", " 'Disabilities',\n", " 'Life Threatening Illnesses',\n", " 'Severe reports',\n", " 'Lethality'\n", " ]]\n", " return summationTable\n", "\n", " @staticmethod\n", " def _flattenColumns(dataFrame):\n", " dataFrame.columns = [\"_\".join(a) for a in dataFrame.columns.to_flat_index()]\n" ] }, { "cell_type": "code", "execution_count": null, "id": "c40bd0f0", "metadata": {}, "outputs": [], "source": [ "import pycountry\n", "\n", "class CountryColumnAdder:\n", " \n", " @staticmethod\n", " def addCountryColumn(dataFrame):\n", " dataFrame['COUNTRY'] = CountryColumnAdder.getCountryColumn(dataFrame)\n", " return dataFrame.astype({'COUNTRY': \"string\"})\n", "\n", " @staticmethod\n", " def getCountryColumn(dataFrame):\n", " return dataFrame.apply(\n", " lambda row:\n", " CountryColumnAdder._getCountryNameOfSplttypeOrDefault(\n", " splttype = row['SPLTTYPE'],\n", " default = 'Unknown Country'),\n", " axis = 'columns')\n", "\n", " @staticmethod\n", " def _getCountryNameOfSplttypeOrDefault(splttype, default):\n", " if not isinstance(splttype, str):\n", " return default\n", " \n", " country = pycountry.countries.get(alpha_2 = splttype[:2])\n", " return country.name if country is not None else default" ] }, { "cell_type": "code", "execution_count": null, "id": "3abe3384", "metadata": {}, "outputs": [], "source": [ "import pycountry\n", "\n", "class SevereColumnAdder:\n", " \n", " @staticmethod\n", " def addSevereColumn(dataFrame):\n", " dataFrame['SEVERE'] = (dataFrame['DIED'] + dataFrame['L_THREAT'] + dataFrame['DISABLE']) > 0\n", " dataFrame['SEVERE'].replace({True: 1, False: 0}, inplace = True)\n", " return dataFrame\n" ] }, { "cell_type": "code", "execution_count": null, "id": "2dad09e5", "metadata": {}, "outputs": [], "source": [ "class CompanyColumnAdder:\n", " \n", " @staticmethod\n", " def addCompanyColumn(batchCodeTable, companyByBatchCodeTable):\n", " return pd.merge(\n", " batchCodeTable,\n", " companyByBatchCodeTable,\n", " how = 'left',\n", " left_index = True,\n", " right_index = True,\n", " validate = 'one_to_one')\n", "\n", " @staticmethod\n", " def createCompanyByBatchCodeTable(dataFrame):\n", " return CompanyColumnAdder._createManufacturerByBatchCodeTable(dataFrame).rename(columns = {\"VAX_MANU\": \"Company\"})\n", "\n", " @staticmethod\n", " def _createManufacturerByBatchCodeTable(dataFrame):\n", " manufacturerByBatchCodeTable = dataFrame[['VAX_LOT', 'VAX_MANU']]\n", " manufacturerByBatchCodeTable = manufacturerByBatchCodeTable.drop_duplicates(subset = ['VAX_LOT'])\n", " return manufacturerByBatchCodeTable.set_index('VAX_LOT')\n" ] }, { "cell_type": "code", "execution_count": null, "id": "09e6b511", "metadata": {}, "outputs": [], "source": [ "class InternationalLotTableFactory:\n", " \n", " def __init__(self, dataFrame : pd.DataFrame):\n", " self.dataFrame = DataFrameFilter().filterByCovid19(dataFrame)\n", " self.batchCodeTableByCountryFactory = BatchCodeTableByCountryFactory(dataFrame)\n", "\n", " def createInternationalLotTable(self):\n", " internationalLotTable = self._createInternationalLotTable()\n", " return internationalLotTable.sort_values(by = 'Severe reports', ascending = False)\n", "\n", " def createBatchCodeTableByCountry(self, country):\n", " return self.batchCodeTableByCountryFactory.createBatchCodeTableByCountry(country)\n", "\n", " def createGlobalBatchCodeTable(self):\n", " return self.createBatchCodeTableByCountry(None)\n", "\n", " def _createInternationalLotTable(self):\n", " return SummationTableFactory.createSummationTableHavingSevereReportsColumn(self.dataFrame.groupby(self.dataFrame['COUNTRY']))\n" ] }, { "cell_type": "code", "execution_count": null, "id": "71456a79", "metadata": {}, "outputs": [], "source": [ "class BatchCodeTableByCountryFactory:\n", "\n", " def __init__(self, dataFrame : pd.DataFrame):\n", " self.dataFrame = DataFrameFilter().filterByCovid19(dataFrame)\n", " self.countryBatchCodeTable = None\n", "\n", " def createBatchCodeTableByCountry(self, country):\n", " batchCodeTable = self._createBatchCodeTableByCountry(country)\n", " batchCodeTable = CompanyColumnAdder.addCompanyColumn(batchCodeTable, CompanyColumnAdder.createCompanyByBatchCodeTable(self.dataFrame))\n", " batchCodeTable = batchCodeTable[\n", " [\n", " 'Adverse Reaction Reports',\n", " 'Deaths',\n", " 'Disabilities',\n", " 'Life Threatening Illnesses',\n", " 'Company',\n", " 'Severe reports',\n", " 'Lethality'\n", " ]]\n", " return batchCodeTable.sort_values(by = 'Severe reports', ascending = False)\n", "\n", " # FK-TODO: refactor\n", " def _createBatchCodeTableByCountry(self, country):\n", " if country is None:\n", " return SummationTableFactory.createSummationTableHavingSevereReportsColumn(self.dataFrame.groupby('VAX_LOT'))\n", "\n", " if self.countryBatchCodeTable is None:\n", " self.countryBatchCodeTable = self._getCountryBatchCodeTable()\n", " \n", " return self._getCountry(self.countryBatchCodeTable, country)\n", "\n", " def _getCountryBatchCodeTable(self):\n", " return SummationTableFactory.createSummationTableHavingSevereReportsColumn(\n", " self.dataFrame.groupby(\n", " [\n", " self.dataFrame['COUNTRY'],\n", " self.dataFrame['VAX_LOT']\n", " ]))\n", "\n", " def _getCountry(self, countryBatchCodeTable, country):\n", " return countryBatchCodeTable.loc[country] if country in countryBatchCodeTable.index else self._getEmptyBatchCodeTable(countryBatchCodeTable)\n", " \n", " def _getEmptyBatchCodeTable(self, countryBatchCodeTable):\n", " return countryBatchCodeTable[0:0].droplevel(0)\n", "\n", " " ] }, { "cell_type": "code", "execution_count": null, "id": "6aa28541", "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "class IOUtils:\n", "\n", " @staticmethod\n", " def saveDataFrame(dataFrame, file):\n", " IOUtils.saveDataFrameAsExcelFile(dataFrame, file)\n", " IOUtils.saveDataFrameAsHtml(dataFrame, file)\n", " IOUtils.saveDataFrameAsJson(dataFrame, file)\n", "\n", " @staticmethod\n", " def saveDataFrameAsExcelFile(dataFrame, file):\n", " IOUtils.ensurePath(file)\n", " dataFrame.to_excel(file + '.xlsx')\n", "\n", " @staticmethod\n", " def saveDataFrameAsHtml(dataFrame, file):\n", " IOUtils.ensurePath(file)\n", " dataFrame.reset_index().to_html(\n", " file + '.html',\n", " index = False,\n", " table_id = 'batchCodeTable',\n", " classes = 'display',\n", " justify = 'unset',\n", " border = 0)\n", "\n", " @staticmethod\n", " def saveDataFrameAsJson(dataFrame, file):\n", " IOUtils.ensurePath(file)\n", " dataFrame.reset_index().to_json(\n", " file + '.json',\n", " orient = \"split\",\n", " index = False)\n", "\n", " @staticmethod\n", " def ensurePath(file):\n", " directory = os.path.dirname(file)\n", " if not os.path.exists(directory):\n", " os.makedirs(directory)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3dacedfd", "metadata": {}, "outputs": [], "source": [ "import unittest" ] }, { "cell_type": "code", "execution_count": null, "id": "fcc855dd", "metadata": {}, "outputs": [], "source": [ "class TestHelper:\n", "\n", " @staticmethod\n", " def createDataFrame(index, columns, data, dtypes = {}):\n", " return pd.DataFrame(index = index, columns = columns, data = data).astype(dtypes)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "ccb9838d", "metadata": {}, "outputs": [], "source": [ "from pandas.testing import assert_frame_equal\n", "\n", "class DataFrameNormalizerTest(unittest.TestCase):\n", "\n", " def test_convertVAX_LOTColumnToUpperCase(self):\n", " # Given\n", " dataFrame = TestHelper.createDataFrame(\n", " columns = ['VAX_LOT'],\n", " data = [ ['037K20A'],\n", " ['025l20A'],\n", " ['025L20A']],\n", " index = [\n", " \"0916600\",\n", " \"0916601\",\n", " \"1996874\"])\n", " \n", " # When\n", " DataFrameNormalizer.convertVAX_LOTColumnToUpperCase(dataFrame)\n", " \n", " # Then\n", " dataFrameExpected = TestHelper.createDataFrame(\n", " columns = ['VAX_LOT'],\n", " data = [ ['037K20A'],\n", " ['025L20A'],\n", " ['025L20A']],\n", " index = [\n", " \"0916600\",\n", " \"0916601\",\n", " \"1996874\"])\n", " assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e59a1825", "metadata": {}, "outputs": [], "source": [ "from pandas.testing import assert_frame_equal\n", "\n", "class DataFrameFilterTest(unittest.TestCase):\n", "\n", " def test_filterByCovid19(self):\n", " # Given\n", " dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(\n", " [\n", " {\n", " 'VAERSDATA': TestHelper.createDataFrame(\n", " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", " data = [ [1, 0, 0],\n", " [0, 0, 1]],\n", " index = [\n", " \"0916600\",\n", " \"0916601\"]),\n", " 'VAERSVAX': TestHelper.createDataFrame(\n", " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['COVID19', 'MODERNA', '037K20A', '1'],\n", " ['COVID19', 'MODERNA', '025L20A', '1']],\n", " index = [\n", " \"0916600\",\n", " \"0916601\"],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " },\n", " {\n", " 'VAERSDATA': TestHelper.createDataFrame(\n", " columns = ['DIED', 'L_THREAT', 'DISABLE'],\n", " data = [ [0, 0, 0],\n", " [0, 0, 1]],\n", " index = [\n", " \"1996873\",\n", " \"1996874\"]),\n", " 'VAERSVAX': TestHelper.createDataFrame(\n", " columns = ['VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ ['HPV9', 'MERCK & CO. INC.', 'R017624', 'UNK'],\n", " ['COVID19', 'MODERNA', '025L20A', '1']],\n", " index = [\n", " \"1996873\",\n", " \"1996874\"],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " }\n", " ])\n", " dataFrameFilter = DataFrameFilter()\n", " \n", " # When\n", " dataFrame = dataFrameFilter.filterByCovid19(dataFrame)\n", " \n", " # Then\n", " dataFrameExpected = TestHelper.createDataFrame(\n", " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES'],\n", " data = [ [1, 0, 0, 'COVID19', 'MODERNA', '037K20A', '1'],\n", " [0, 0, 1, 'COVID19', 'MODERNA', '025L20A', '1'],\n", " [0, 0, 1, 'COVID19', 'MODERNA', '025L20A', '1']],\n", " index = [\n", " \"0916600\",\n", " \"0916601\",\n", " \"1996874\"],\n", " dtypes = {'VAX_DOSE_SERIES': \"string\"})\n", " assert_frame_equal(dataFrame, dataFrameExpected, check_dtype = False)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "c784bfef", "metadata": {}, "outputs": [], "source": [ "from pandas.testing import assert_frame_equal\n", "\n", "class InternationalLotTableFactoryTest(unittest.TestCase):\n", "\n", " def test_createInternationalLotTable(self):\n", " # Given\n", " dataFrame = TestHelper.createDataFrame(\n", " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES', 'SPLTTYPE', 'HOSPITAL', 'ER_VISIT', 'COUNTRY'],\n", " data = [ [1, 0, 0, 'COVID19', 'MODERNA', '016M20A', '2', 'GBPFIZER INC2020486806', 0, 0, 'United Kingdom'],\n", " [1, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France'],\n", " [1, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France'],\n", " [0, 0, 0, 'COVID19', 'MODERNA', '030L20B', '1', 'dummy', 0, 0, 'Unknown Country'],\n", " [0, 0, 0, 'COVID19', 'MODERNA', '030L20B', '1', 123, 0, 0, 'Unknown Country']],\n", " index = [\n", " \"1048786\",\n", " \"1048786\",\n", " \"4711\",\n", " \"0815\",\n", " \"0816\"])\n", " dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n", " internationalLotTableFactory = InternationalLotTableFactory(dataFrame)\n", " \n", " # When\n", " internationalLotTable = internationalLotTableFactory.createInternationalLotTable()\n", "\n", " # Then\n", " assert_frame_equal(\n", " internationalLotTable,\n", " TestHelper.createDataFrame(\n", " columns = ['Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Severe reports', 'Lethality'],\n", " data = [ [2, 2, 1, 1, 2/2 * 100, 2/2 * 100],\n", " [1, 1, 0, 0, 1/1 * 100, 1/1 * 100],\n", " [2, 0, 0, 0, 0/2 * 100, 0/2 * 100]],\n", " index = pd.Index(\n", " [\n", " 'France',\n", " 'United Kingdom',\n", " 'Unknown Country'\n", " ],\n", " name = 'COUNTRY')))\n", "\n", " def test_createBatchCodeTableByCountry(self):\n", " # Given\n", " dataFrame = TestHelper.createDataFrame(\n", " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES', 'SPLTTYPE', 'HOSPITAL', 'ER_VISIT', 'COUNTRY'],\n", " data = [ [1, 0, 0, 'COVID19', 'PFIZER\\BIONTECH', '016M20A', '2', 'GBPFIZER INC2020486806', 0, 0, 'United Kingdom'],\n", " [0, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France'],\n", " [1, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France'],\n", " [0, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France']],\n", " index = [\n", " \"1048786\",\n", " \"1048786\",\n", " \"4711\",\n", " \"0815\"])\n", " dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n", " internationalLotTableFactory = InternationalLotTableFactory(dataFrame)\n", " \n", " # When\n", " batchCodeTable = internationalLotTableFactory.createBatchCodeTableByCountry('France')\n", "\n", " # Then\n", " assert_frame_equal(\n", " batchCodeTable,\n", " TestHelper.createDataFrame(\n", " columns = ['Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Company', 'Severe reports', 'Lethality'],\n", " data = [ [2, 1, 2, 2, 'MODERNA', 2/2 * 100, 1/2 * 100],\n", " [1, 0, 0, 0, 'MODERNA', 0/1 * 100, 0/1 * 100]],\n", " index = pd.Index(\n", " [\n", " '030L20B',\n", " '030L20A'\n", " ],\n", " name = 'VAX_LOT')),\n", " check_dtype = False)\n", "\n", " def test_createGlobalBatchCodeTable(self):\n", " # Given\n", " dataFrame = TestHelper.createDataFrame(\n", " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES', 'SPLTTYPE', 'HOSPITAL', 'ER_VISIT', 'COUNTRY'],\n", " data = [ [1, 0, 0, 'COVID19', 'PFIZER\\BIONTECH', '016M20A', '2', 'GBPFIZER INC2020486806', 0, 0, 'United Kingdom'],\n", " [0, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France'],\n", " [1, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France'],\n", " [0, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France']],\n", " index = [\n", " \"1048786\",\n", " \"1048786\",\n", " \"4711\",\n", " \"0815\"])\n", " dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n", " internationalLotTableFactory = InternationalLotTableFactory(dataFrame)\n", " \n", " # When\n", " batchCodeTable = internationalLotTableFactory.createGlobalBatchCodeTable()\n", "\n", " # Then\n", " assert_frame_equal(\n", " batchCodeTable,\n", " TestHelper.createDataFrame(\n", " columns = ['Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Company', 'Severe reports', 'Lethality'],\n", " data = [ [1, 1, 0, 0, 'PFIZER\\BIONTECH', 1/1 * 100, 1/1 * 100],\n", " [2, 1, 2, 2, 'MODERNA', 2/2 * 100, 1/2 * 100],\n", " [1, 0, 0, 0, 'MODERNA', 0/1 * 100, 0/1 * 100]],\n", " index = pd.Index(\n", " [\n", " '016M20A',\n", " '030L20B',\n", " '030L20A'\n", " ],\n", " name = 'VAX_LOT')),\n", " check_dtype = False)\n", "\n", " def test_createBatchCodeTableByNonExistingCountry(self):\n", " # Given\n", " dataFrame = TestHelper.createDataFrame(\n", " columns = ['DIED', 'L_THREAT', 'DISABLE', 'VAX_TYPE', 'VAX_MANU', 'VAX_LOT', 'VAX_DOSE_SERIES', 'SPLTTYPE', 'HOSPITAL', 'ER_VISIT', 'COUNTRY'],\n", " data = [ [1, 0, 0, 'COVID19', 'PFIZER\\BIONTECH', '016M20A', '2', 'GBPFIZER INC2020486806', 0, 0, 'United Kingdom'],\n", " [0, 0, 0, 'COVID19', 'MODERNA', '030L20A', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France'],\n", " [1, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France'],\n", " [0, 1, 1, 'COVID19', 'MODERNA', '030L20B', '1', 'FRMODERNATX, INC.MOD20224', 0, 0, 'France']],\n", " index = [\n", " \"1048786\",\n", " \"1048786\",\n", " \"4711\",\n", " \"0815\"])\n", " dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n", " internationalLotTableFactory = InternationalLotTableFactory(dataFrame)\n", " \n", " # When\n", " batchCodeTable = internationalLotTableFactory.createBatchCodeTableByCountry('non existing country')\n", "\n", " # Then\n", " assert_frame_equal(\n", " batchCodeTable,\n", " TestHelper.createDataFrame(\n", " columns = ['Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Company', 'Severe reports', 'Lethality'],\n", " data = [ ],\n", " index = pd.Index([], name = 'VAX_LOT')),\n", " check_dtype = False)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5a8bff1b", "metadata": {}, "outputs": [], "source": [ "unittest.main(argv = [''], verbosity = 2, exit = False)" ] }, { "cell_type": "code", "execution_count": null, "id": "86e0e4f2", "metadata": {}, "outputs": [], "source": [ "def getVaersForYear(year):\n", " return getVaersForYears([year])\n", "\n", "def getVaersForYears(years):\n", " def addCountryColumn(dataFrame):\n", " dataFrame['COUNTRY'] = 'United States'\n", " return dataFrame\n", "\n", " return _getVaers(\n", " _getVaersDescrReader().readVaersDescrsForYears(years),\n", " addCountryColumn)\n", "\n", "def getNonDomesticVaers():\n", " return _getVaers(\n", " [_getVaersDescrReader().readNonDomesticVaersDescr()],\n", " CountryColumnAdder.addCountryColumn)\n", "\n", "def _getVaersDescrReader():\n", " return VaersDescrReader(dataDir = \"VAERS\")\n", "\n", "def _getVaers(vaersDescrs, addCountryColumn):\n", " dataFrame = VaersDescr2DataFrameConverter.createDataFrameFromDescrs(vaersDescrs)\n", " dataFrame = addCountryColumn(dataFrame)\n", " DataFrameNormalizer.normalize(dataFrame)\n", " dataFrame = SevereColumnAdder.addSevereColumn(dataFrame)\n", " return dataFrame\n", " " ] }, { "cell_type": "code", "execution_count": null, "id": "8e3bc9b3", "metadata": {}, "outputs": [], "source": [ "vaers = getVaersForYears([2020, 2021, 2022])\n", "vaers" ] }, { "cell_type": "code", "execution_count": null, "id": "fbf5006a", "metadata": {}, "outputs": [], "source": [ "nonDomesticVaers = getNonDomesticVaers()\n", "nonDomesticVaers" ] }, { "cell_type": "code", "execution_count": null, "id": "781ac80e", "metadata": {}, "outputs": [], "source": [ "internationalVaers = pd.concat([vaers, nonDomesticVaers])\n", "internationalVaers" ] }, { "cell_type": "code", "execution_count": null, "id": "ff259a35", "metadata": {}, "outputs": [], "source": [ "def createAndSaveBatchCodeTableForCountry(createBatchCodeTableForCountry, country, minADRsForLethality = None):\n", " batchCodeTable = createBatchCodeTableForCountry(country)\n", " batchCodeTable.index.set_names(\"Batch\", inplace = True)\n", " if minADRsForLethality is not None:\n", " batchCodeTable.loc[batchCodeTable['Adverse Reaction Reports'] < minADRsForLethality, ['Severe reports', 'Lethality']] = [np.nan, np.nan]\n", " IOUtils.saveDataFrame(batchCodeTable, '../docs/data/' + country)\n", " display(country + \":\", batchCodeTable)\n", "\n", "def createAndSaveBatchCodeTablesForCountries(createBatchCodeTableForCountry, countries, minADRsForLethality = None):\n", " for country in countries:\n", " createAndSaveBatchCodeTableForCountry(createBatchCodeTableForCountry, country, minADRsForLethality)" ] }, { "cell_type": "code", "execution_count": null, "id": "cc1ef82a", "metadata": {}, "outputs": [], "source": [ "def printCountryOptions(countries):\n", " for country in countries:\n", " printCountryOption(country)\n", "\n", "def printCountryOption(country):\n", " print(''.format(country = country))" ] }, { "cell_type": "code", "execution_count": null, "id": "0c4d04fb", "metadata": {}, "outputs": [], "source": [ "countries = sorted(internationalVaers['COUNTRY'].unique())\n", "printCountryOptions(countries)" ] }, { "cell_type": "code", "execution_count": null, "id": "7e7e01a5", "metadata": {}, "outputs": [], "source": [ "minADRsForLethality = 100\n", "internationalLotTableFactory = InternationalLotTableFactory(internationalVaers)\n", "\n", "createAndSaveBatchCodeTablesForCountries(\n", " createBatchCodeTableForCountry = lambda country: internationalLotTableFactory.createBatchCodeTableByCountry(country),\n", " countries = countries,\n", " minADRsForLethality = minADRsForLethality)\n", "\n", "createAndSaveBatchCodeTableForCountry(\n", " createBatchCodeTableForCountry = lambda country: internationalLotTableFactory.createGlobalBatchCodeTable(),\n", " country = 'Global',\n", " minADRsForLethality = minADRsForLethality)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "376b9193", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.7" } }, "nbformat": 4, "nbformat_minor": 5 }