francois/HowBadIsMyBatch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactoring

Browse Source
This commit is contained in:
frankknoll
2022-02-20 12:33:21 +01:00
parent be6ee46445
commit a0a05bfed9
496 changed files with 396 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
src/HowBadIsMyBatch.ipynb
View File

@@ -182,8 +182,7 @@
" def createSummationTable(\n",
" groupBy,\n",
" columnNameMappingsDict = {\n",
" # FK-TODO: rename \"Total Number of Adverse Reaction Reports\" to \"Adverse Reaction Reports\" everywhere\n",
" \"DIED_size\": \"Total Number of Adverse Reaction Reports\",\n",
" \"DIED_size\": \"Adverse Reaction Reports\",\n",
" \"DIED_sum\": \"Deaths\",\n",
" \"L_THREAT_sum\": \"Life Threatening Illnesses\",\n",
" \"DISABLE_sum\": \"Disabilities\",\n",
@@ -207,17 +206,17 @@
" summationTable = SummationTableFactory.createSummationTable(\n",
" dataFrame,\n",
" columnNameMappingsDict = {\n",
" \"DIED_size\": \"Total Number of Adverse Reaction Reports\",\n",
" \"DIED_size\": \"Adverse Reaction Reports\",\n",
" \"DIED_sum\": \"Deaths\",\n",
" \"L_THREAT_sum\": \"Life Threatening Illnesses\",\n",
" \"DISABLE_sum\": \"Disabilities\",\n",
" \"SEVERE_sum\": \"Severities\"\n",
" })\n",
" summationTable['Severe reports'] = summationTable['Severities'] / summationTable['Total Number of Adverse Reaction Reports'] * 100\n",
" summationTable['Lethality'] = summationTable['Deaths'] / summationTable['Total Number of Adverse Reaction Reports'] * 100\n",
" summationTable['Severe reports'] = summationTable['Severities'] / summationTable['Adverse Reaction Reports'] * 100\n",
" summationTable['Lethality'] = summationTable['Deaths'] / summationTable['Adverse Reaction Reports'] * 100\n",
" summationTable = summationTable[\n",
" [\n",
" 'Total Number of Adverse Reaction Reports',\n",
" 'Adverse Reaction Reports',\n",
" 'Deaths',\n",
" 'Disabilities',\n",
" 'Life Threatening Illnesses',\n",
@@ -248,7 +247,7 @@
" dataFrame = DataFrameFilter().filterBy(dataFrame, dose = dose)\n",
" batchCodeTable = BatchCodeTableFactory._createSummationTableByVAX_LOT(dataFrame)[\n",
" [\n",
" 'Total Number of Adverse Reaction Reports',\n",
" 'Adverse Reaction Reports',\n",
" 'Deaths',\n",
" 'Disabilities',\n",
" 'Life Threatening Illnesses',\n",
@@ -256,7 +255,7 @@
" 'Lethality'\n",
" ]]\n",
" if minADRsForLethality is not None:\n",
" batchCodeTable.loc[batchCodeTable['Total Number of Adverse Reaction Reports'] < minADRsForLethality, 'Lethality'] = np.nan\n",
" batchCodeTable.loc[batchCodeTable['Adverse Reaction Reports'] < minADRsForLethality, 'Lethality'] = np.nan\n",
" return batchCodeTable\n",
"\n",
" # create table from https://www.howbadismybatch.com/combined.html\n",
@@ -266,7 +265,7 @@
" dataFrame = DataFrameFilter().filterBy(dataFrame, dose = dose)\n",
" return BatchCodeTableFactory._createSummationTableByVAX_LOT(dataFrame)[\n",
" [\n",
" 'Total Number of Adverse Reaction Reports', \n",
" 'Adverse Reaction Reports', \n",
" 'Deaths',\n",
" 'Disabilities',\n",
" 'Life Threatening Illnesses',\n",
@@ -278,10 +277,10 @@
" @staticmethod\n",
" def _createSummationTableByVAX_LOT(dataFrame):\n",
" batchCodeTable = SummationTableFactory.createSummationTable(dataFrame.groupby('VAX_LOT'))\n",
" batchCodeTable['Lethality'] = batchCodeTable['Deaths'] / batchCodeTable['Total Number of Adverse Reaction Reports'] * 100\n",
" batchCodeTable['Lethality'] = batchCodeTable['Deaths'] / batchCodeTable['Adverse Reaction Reports'] * 100\n",
" batchCodeTable = batchCodeTable[\n",
" [\n",
" 'Total Number of Adverse Reaction Reports',\n",
" 'Adverse Reaction Reports',\n",
" 'Deaths',\n",
" 'Disabilities',\n",
" 'Life Threatening Illnesses',\n",
@@ -289,7 +288,7 @@
" 'Emergency Room or Doctor Visits',\n",
" 'Lethality'\n",
" ]]\n",
" batchCodeTable = batchCodeTable.sort_values(by = 'Total Number of Adverse Reaction Reports', ascending = False)\n",
" batchCodeTable = batchCodeTable.sort_values(by = 'Adverse Reaction Reports', ascending = False)\n",
" return CompanyColumnAdder.addCompanyColumn(batchCodeTable, CompanyColumnAdder.createCompanyByBatchCodeTable(dataFrame))\n"
]
},
@@ -425,7 +424,7 @@
" batchCodeTable = CompanyColumnAdder.addCompanyColumn(batchCodeTable, CompanyColumnAdder.createCompanyByBatchCodeTable(dataFrame))\n",
" batchCodeTable = batchCodeTable[\n",
" [\n",
" 'Total Number of Adverse Reaction Reports',\n",
" 'Adverse Reaction Reports',\n",
" 'Deaths',\n",
" 'Disabilities',\n",
" 'Life Threatening Illnesses',\n",
@@ -868,7 +867,7 @@
" # Then\n",
" batchCodeTableExpected = pd.DataFrame(\n",
" data = {\n",
" 'Total Number of Adverse Reaction Reports': [1, 1],\n",
" 'Adverse Reaction Reports': [1, 1],\n",
" 'Deaths': [0, 1],\n",
" 'Disabilities': [1, 0],\n",
" 'Life Threatening Illnesses': [0, 1],\n",
@@ -967,12 +966,12 @@
" # Then\n",
" batchCodeTableExpected = pd.DataFrame(\n",
" data = {\n",
" 'Total Number of Adverse Reaction Reports': [2, 1],\n",
" 'Deaths': [0, 1],\n",
" 'Disabilities': [2, 0],\n",
" 'Life Threatening Illnesses': [0, 0],\n",
" 'Company': ['MODERNA', 'MODERNA'],\n",
" 'Lethality': [0/2 * 100, np.nan]\n",
" 'Adverse Reaction Reports': [2, 1],\n",
" 'Deaths': [0, 1],\n",
" 'Disabilities': [2, 0],\n",
" 'Life Threatening Illnesses': [0, 0],\n",
" 'Company': ['MODERNA', 'MODERNA'],\n",
" 'Lethality': [0/2 * 100, np.nan]\n",
" },\n",
" index = pd.Index(['025L20A', '037K20A'], name = 'VAX_LOT'))\n",
" assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n",
@@ -992,12 +991,12 @@
" # Then\n",
" batchCodeTableExpected = pd.DataFrame(\n",
" data = {\n",
" 'Total Number of Adverse Reaction Reports': [2, 1],\n",
" 'Deaths': [0, 1],\n",
" 'Disabilities': [2, 0],\n",
" 'Life Threatening Illnesses': [0, 0],\n",
" 'Company': ['MODERNA', 'MODERNA'],\n",
" 'Lethality': [0/2 * 100, 1/1 * 100]\n",
" 'Adverse Reaction Reports': [2, 1],\n",
" 'Deaths': [0, 1],\n",
" 'Disabilities': [2, 0],\n",
" 'Life Threatening Illnesses': [0, 0],\n",
" 'Company': ['MODERNA', 'MODERNA'],\n",
" 'Lethality': [0/2 * 100, 1/1 * 100]\n",
" },\n",
" index = pd.Index(['025L20A', '037K20A'], name = 'VAX_LOT'))\n",
" assert_frame_equal(batchCodeTable, batchCodeTableExpected, check_dtype = False)\n"
@@ -1036,12 +1035,12 @@
" doseTable,\n",
" pd.DataFrame(\n",
" data = {\n",
" 'Total Number of Adverse Reaction Reports': [2, 1],\n",
" 'Deaths': [2, 1],\n",
" 'Disabilities': [1, 0],\n",
" 'Life Threatening Illnesses': [1, 0],\n",
" 'Severe reports': [2/2 * 100, 1/1 * 100],\n",
" 'Lethality': [2/2 * 100, 1/1 * 100]\n",
" 'Adverse Reaction Reports': [2, 1],\n",
" 'Deaths': [2, 1],\n",
" 'Disabilities': [1, 0],\n",
" 'Life Threatening Illnesses': [1, 0],\n",
" 'Severe reports': [2/2 * 100, 1/1 * 100],\n",
" 'Lethality': [2/2 * 100, 1/1 * 100]\n",
" },\n",
" index = pd.Index(['1', '2'], dtype = \"string\", name = 'Dose')))\n",
" \n",
@@ -1068,12 +1067,12 @@
" doseByMonthTable,\n",
" pd.DataFrame(\n",
" data = {\n",
" 'Total Number of Adverse Reaction Reports': [2, 1],\n",
" 'Deaths': [2, 1],\n",
" 'Disabilities': [1, 0],\n",
" 'Life Threatening Illnesses': [1, 0],\n",
" 'Severe reports': [2/2 * 100, 1/1 * 100],\n",
" 'Lethality': [2/2 * 100, 1/1 * 100]\n",
" 'Adverse Reaction Reports': [2, 1],\n",
" 'Deaths': [2, 1],\n",
" 'Disabilities': [1, 0],\n",
" 'Life Threatening Illnesses': [1, 0],\n",
" 'Severe reports': [2/2 * 100, 1/1 * 100],\n",
" 'Lethality': [2/2 * 100, 1/1 * 100]\n",
" },\n",
" index = pd.MultiIndex.from_tuples(\n",
" [\n",
@@ -1119,10 +1118,10 @@
" assert_frame_equal(\n",
" internationalLotTable,\n",
" TestHelper.createDataFrame(\n",
" columns = ['Total Number of Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Severe reports', 'Lethality'],\n",
" data = [ [2, 2, 1, 1, 2/2 * 100, 2/2 * 100],\n",
" [1, 1, 0, 0, 1/1 * 100, 1/1 * 100],\n",
" [2, 0, 0, 0, 0/2 * 100, 0/2 * 100]],\n",
" columns = ['Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Severe reports', 'Lethality'],\n",
" data = [ [2, 2, 1, 1, 2/2 * 100, 2/2 * 100],\n",
" [1, 1, 0, 0, 1/1 * 100, 1/1 * 100],\n",
" [2, 0, 0, 0, 0/2 * 100, 0/2 * 100]],\n",
" index = pd.Index(\n",
" [\n",
" 'France',\n",
@@ -1154,9 +1153,9 @@
" assert_frame_equal(\n",
" batchCodeTable,\n",
" TestHelper.createDataFrame(\n",
" columns = ['Total Number of Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Company', 'Severe reports', 'Lethality'],\n",
" data = [ [2, 1, 2, 2, 'MODERNA', 2/2 * 100, 1/2 * 100],\n",
" [1, 0, 0, 0, 'MODERNA', 0/1 * 100, 0/1 * 100]],\n",
" columns = ['Adverse Reaction Reports', 'Deaths', 'Disabilities', 'Life Threatening Illnesses', 'Company', 'Severe reports', 'Lethality'],\n",
" data = [ [2, 1, 2, 2, 'MODERNA', 2/2 * 100, 1/2 * 100],\n",
" [1, 0, 0, 0, 'MODERNA', 0/1 * 100, 0/1 * 100]],\n",
" index = pd.Index(\n",
" [\n",
" '030L20B',\n",
@@ -1263,7 +1262,7 @@
"outputs": [],
"source": [
"def saveBatchCodeTable(vaers, file):\n",
" # FK-TODO: show \"Sever Reports\"\n",
" # FK-TODO: show \"Severe Reports\"\n",
" # FK-TODO: show combination of ALL doses\n",
" batchCodeTable = BatchCodeTableFactory.createBatchCodeTable(vaers, dose = '1', minADRsForLethality = 100)\n",
" batchCodeTable.index.set_names(\"Batch\", inplace = True)\n",
@@ -1379,7 +1378,7 @@
"metadata": {},
"outputs": [],
"source": [
"internationalLotTable = internationalLotTable[internationalLotTable['Total Number of Adverse Reaction Reports'] > 50]\n",
"internationalLotTable = internationalLotTable[internationalLotTable['Adverse Reaction Reports'] > 50]\n",
"IOUtils.saveDataFrame(internationalLotTable, 'results/international/International_Deadly_Lots')\n",
"internationalLotTable"
]
@@ -1395,7 +1394,7 @@
" batchCodeTable = InternationalLotTableFactory.createBatchCodeTableByCountry(nonDomesticVaers, country)\n",
" batchCodeTable.index.set_names(\"Batch\", inplace = True)\n",
" if minADRsForLethality is not None:\n",
" batchCodeTable.loc[batchCodeTable['Total Number of Adverse Reaction Reports'] < minADRsForLethality, ['Severe reports', 'Lethality']] = [np.nan, np.nan]\n",
" batchCodeTable.loc[batchCodeTable['Adverse Reaction Reports'] < minADRsForLethality, ['Severe reports', 'Lethality']] = [np.nan, np.nan]\n",
" IOUtils.saveDataFrame(batchCodeTable, '../docs/data/' + country)\n",
" display(country + \":\", batchCodeTable)\n",
"\n",
@@ -1486,7 +1485,7 @@
"def createADRsByVAX_LOTTable(vaers, manufacturer):\n",
" dataFrame = DataFrameFilter().filterByCovid19(vaers)\n",
" dataFrame = DataFrameFilter().filterBy(dataFrame, manufacturer = manufacturer)\n",
" batchCodeTable = BatchCodeTableFactory._createSummationTableByVAX_LOT(dataFrame)[['Total Number of Adverse Reaction Reports']].reset_index()\n",
" batchCodeTable = BatchCodeTableFactory._createSummationTableByVAX_LOT(dataFrame)[['Adverse Reaction Reports']].reset_index()\n",
" return batchCodeTable\n",
"\n",
"def filterColumnOfDataFrameWithRegexp(dataFrame, column, regexp):\n",
@@ -1508,7 +1507,7 @@
"twoLetterPrefix = re.compile(r'^[a-zA-Z]{2}')\n",
"batchCodeTable = filterColumnOfDataFrameWithRegexp(dataFrame = batchCodeTable, column = 'VAX_LOT_PREFIX', regexp = twoLetterPrefix)\n",
"batchCodeTable = batchCodeTable[batchCodeTable['VAX_LOT_PREFIX'].isin(['EN', 'EP', 'ER', 'EW', 'FA', 'FC', 'FD', 'FE', 'FH'])]\n",
"batchCodeTable = batchCodeTable[batchCodeTable['Total Number of Adverse Reaction Reports'] > 400]\n",
"batchCodeTable = batchCodeTable[batchCodeTable['Adverse Reaction Reports'] > 400]\n",
"batchCodeTable"
]
},
@@ -1523,7 +1522,7 @@
"\n",
"sns.set(rc = {'figure.figsize': (11.7, 8.27)})\n",
"sns.set_theme()\n",
"chart = sns.stripplot(x = \"VAX_LOT_PREFIX\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable)"
"chart = sns.stripplot(x = \"VAX_LOT_PREFIX\", y = \"Adverse Reaction Reports\", data = batchCodeTable)"
]
},
{
@@ -1533,7 +1532,7 @@
"metadata": {},
"outputs": [],
"source": [
"sns.pointplot(x = \"VAX_LOT_PREFIX\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable, estimator = np.mean)"
"sns.pointplot(x = \"VAX_LOT_PREFIX\", y = \"Adverse Reaction Reports\", data = batchCodeTable, estimator = np.mean)"
]
},
{
@@ -1546,7 +1545,7 @@
"import seaborn as sns\n",
"sns.set_theme(style = \"ticks\", palette = \"pastel\")\n",
"\n",
"sns.boxplot(x = \"VAX_LOT_PREFIX\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable)"
"sns.boxplot(x = \"VAX_LOT_PREFIX\", y = \"Adverse Reaction Reports\", data = batchCodeTable)"
]
},
{
@@ -1571,7 +1570,7 @@
"batchCodeTable = filterColumnOfDataFrameWithRegexp(dataFrame = batchCodeTable, column = 'VAX_LOT', regexp = modernaBatchCodePrefix)\n",
"batchCodeTable['CONCENTRATION'] = batchCodeTable['VAX_LOT'].str[3]\n",
"batchCodeTable = batchCodeTable.sort_values(by = 'CONCENTRATION', ascending = True)\n",
"batchCodeTable = batchCodeTable[batchCodeTable['Total Number of Adverse Reaction Reports'] > 400]\n",
"batchCodeTable = batchCodeTable[batchCodeTable['Adverse Reaction Reports'] > 400]\n",
"batchCodeTable"
]
},
@@ -1588,7 +1587,7 @@
"\n",
"sns.set(rc = {'figure.figsize': (11.7, 8.27)})\n",
"sns.set_theme()\n",
"chart = sns.stripplot(x = \"CONCENTRATION\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable, order = order)"
"chart = sns.stripplot(x = \"CONCENTRATION\", y = \"Adverse Reaction Reports\", data = batchCodeTable, order = order)"
]
},
{
@@ -1598,7 +1597,7 @@
"metadata": {},
"outputs": [],
"source": [
"sns.pointplot(x = \"CONCENTRATION\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable, estimator = np.mean, order = order)"
"sns.pointplot(x = \"CONCENTRATION\", y = \"Adverse Reaction Reports\", data = batchCodeTable, estimator = np.mean, order = order)"
]
},
{
@@ -1611,7 +1610,7 @@
"import seaborn as sns\n",
"sns.set_theme(style = \"ticks\", palette = \"pastel\")\n",
"\n",
"sns.boxplot(x = \"CONCENTRATION\", y = \"Total Number of Adverse Reaction Reports\", data = batchCodeTable, order = order)"
"sns.boxplot(x = \"CONCENTRATION\", y = \"Adverse Reaction Reports\", data = batchCodeTable, order = order)"
]
},
{
@@ -1649,7 +1648,7 @@
"def _getCovid19BatchCodeTable(dataFrame):\n",
" batchCodeTable = BatchCodeTableFactory._createSummationTableByVAX_LOT(dataFrame)[\n",
" [\n",
" 'Total Number of Adverse Reaction Reports',\n",
" 'Adverse Reaction Reports',\n",
" 'Deaths',\n",
" 'Disabilities',\n",
" 'Life Threatening Illnesses',\n",
@@ -1695,7 +1694,7 @@
"#\n",
"#sns.set(rc = {'figure.figsize': (11.7, 8.27)})\n",
"#sns.set_theme()\n",
"#chart = sns.stripplot(x = \"VAX_LOT\", y = \"Total Number of Adverse Reaction Reports\", data = covid19PfizerBatchCodeTable)"
"#chart = sns.stripplot(x = \"VAX_LOT\", y = \"Adverse Reaction Reports\", data = covid19PfizerBatchCodeTable)"
]
},
{