Merge branch 'develop'

cypress/fixtures/comparison/control.8.9.4.txt

@@ -1,4 +1,5 @@
 ID	ZT3	ZT6	ZT9	ZT3	ZT21	ZT18	ZT24	ZT6	ZT0	ZT12	ZT15	ZT0	ZT9	ZT21	ZT21	ZT3	ZT12	ZT9	ZT6	ZT15	ZT9	ZT0	ZT15	ZT21	ZT15	ZT18	ZT3	ZT12	ZT24	ZT24	ZT18	ZT18	ZT12	ZT6	ZT0	ZT24
+4_cyc	71.28850087120433	73.08393898757465	75.52580519088211	71.184525544128	70.19094024878467	77.92471469243831	71.56461577793065	77.12146929999625	68.84713700563974	74.01872089379407	71.63693500676479	71.44293155497158	70.13377128857915	72.35956743668068	69.55139157302412	74.87749855783053	72.77491748872337	68.78270756060313	72.12159057013677	75.3361084755841	73.85445124580836	73.10672353549445	76.46924100439327	70.18199817335699	72.95964431657626	74.87826693300916	74.05925935781327	70.25587167552806	70.20084505744062	72.5916188785937	75.95255498968604	74.66529548911534	70.0580077563037	71.02329518775886	72.0558836082044	70.2123656267451
 0_cyc	24.457616300295765	22.575764150485004	16.84320383166575	23.89201959242602	27.118174424642586	28.737929818029432	29.661284951395746	18.091454010083215	29.176261220711726	19.895405848487417	23.253612911910306	28.149781888960018	19.92690890227966	29.917425563245867	31.49314585335809	23.89764899424006	20.882053479365812	21.299083870647568	22.126004778399352	19.577954696677825	14.782978912686726	27.67676669395477	20.505372887428965	33.60126379018409	24.911786817847823	28.103685738074628	27.54045829791816	18.8659739768749	27.34939915802379	27.179012422801666	28.56407071152216	29.63655285394395	23.145462182665103	22.807484862031973	26.536771396190968	30.066493112809187
 1_cyc					-39.21201442650625						-37.640717611862854		-40.62786170794967				-38.48738431996823		-41.057673862055346								-42.301419599345344				-38.376216297002514				-42.79742893228427	-41.504575950578555
 2_cyc		-99.24192654384835	-99.41828573561182		-103.89479562934181	-105.21034621435943	-101.47190576046282	-99.56685756855042	-100.38202291611064	-103.97047332216273	-103.80139336443318	-102.59822249299567	-99.17327584972922	-104.43620891335492	-104.97638577409762		-101.88026502707356	-100.7796280651412	-101.10924861583665	-103.14552865726725	-99.85970779098756	-102.28605274653808	-103.43815940406103	-103.11952221428615	-103.71864871400597	-105.5049028958092		-102.2915252694532	-102.5582597889558	-101.47174914199303	-106.73203022248858	-105.3988535944696	-102.05805301923485	-97.8075207691708	-101.21763830433908	-101.5589517465117
@@ -12,9 +13,11 @@ ID	ZT3	ZT6	ZT9	ZT3	ZT21	ZT18	ZT24	ZT6	ZT0	ZT12	ZT15	ZT0	ZT9	ZT21	ZT21	ZT3	ZT12	Z
 10_null	10.991869494749944	10.204058004129928	12.251952133382757	12.542933723626653	12.08271018738517	9.962547238909648	10.742288349010451	11.698728044829453	10.66169869960436	10.66237272926656	11.65009161689141	10.524122407435978	12.39213472991345	9.922587985169427	12.01001782485393	11.783789288497049	10.939116174055513	10.687941654385526	11.552027604269805	11.996024468382792	12.99934688956932	10.705086182529143	8.133555627684729	12.538770858747414	11.336405219634212	10.785344679356156	9.939296707564958	12.25815568012157	10.530835815089029	10.266316944361249	11.344510857416013	10.983541690683182	10.921629053707596	11.426213315481519	11.656555623360955	11.988483932144621
 11_null					-148.67802301195562						-148.96073262549746		-147.1251582754468				-148.254565685753		-147.56536578745462								-149.70503891143977				-150.68952158582505				-149.42125557099757	-147.67975474622685
 12_null		-48.121894669929766	-48.27743154008449		-48.70506980829354	-48.70866548155434	-46.663168244274374	-48.09443235063856	-48.25441456243834	-48.55721175659582	-48.367666001605016	-47.59826445141418	-47.24770226459147	-49.16370509409916	-47.38576327584462		-46.956962733422294	-48.2348785564673	-46.38865447687907	-47.185842734437195	-46.6988876432822	-50.174890214953116	-46.387920502226336	-46.50699618036778	-48.83322595333884	-47.758477288104494		-48.53005388780452	-47.42399729184897	-46.10688212612639	-47.11517644857655	-48.99126370895602	-47.61987793596584	-50.05712368321645	-48.62245692113646	-46.92569552701677
+7_cyc	-13.251061625030538	-17.84856457824676	-16.75985346763388	-13.520339206523186	-12.369409999722746	-14.557907412982422	-10.865548740580833	-17.544721700740382	-12.930843351639204	-17.37736500020131	-16.924646634787212	-14.342905156517054	-16.09138569413995	-12.97669843539518	-13.407320403124075	-13.923297863879124	-17.555771368134078	-18.1422936298538	-16.899390191597426	-16.019294760478374	-17.947092574558468	-10.925726392024673	-17.493565657086577	-12.603385581379797	-14.91635690164124	-13.396224878777787	-13.100699342846472	-16.796982184165348	-10.502187747704303	-11.768606020005286	-12.838258757591223	-13.74838028862793	-18.040901696305855	-15.936173887350725	-11.827714647992112	-14.032716890255521
 13_null	184.1313436168848	183.70103510725258	183.07497008252267		183.47947102759179	183.15163198038948	182.08805214684472	182.33432765766003	183.3477217367277	182.10044975204664	181.79723058689223		184.63509058953596	183.66022822079012	183.1331443058404	184.57793547489706	182.77889303073485	185.04568943645037	182.52314426109965	186.08912193658043	182.52845543844043	183.74597670403858	182.91968373172503	184.08285569635262	183.57469784782043	181.3399823381078	182.65257059148462	183.0476538757754	182.81785959783394	182.7094056151721	182.03417987818918	182.8412269141135	182.3785755145488	183.72441955595585	182.71561991677737	183.74128608913617
 14_null	-80.5132064830133	-78.96657278846814	-75.97634875746387	-78.74356940467456	-79.82840432107555	-78.92950016158798	-78.1599116986391	-77.67013858795909	-78.7952488626428	-77.98321582638133	-78.26693556339096	-78.44615756082415	-81.0449046528129	-81.11962756497772	-78.91546995463985	-79.9619986831251	-80.75013905950159	-78.77752000002295	-77.83798083160512	-78.89322033231191	-79.33929652348654	-77.94347058066089	-77.61987175085686	-78.5287077421279	-78.05444888854021	-78.79566894969997	-77.02738913210521	-78.8909305464825	-79.39403242105818	-78.72489874677267	-79.30165752612892	-80.40925936731726	-77.80683752194662	-81.64981563713485	-77.87006622693562	-78.9588243863608
 15_null	-144.4891143156159	-144.08260478647932	-145.0319072095593	-144.285023723934	-142.43864243765935	-143.6893194827693	-143.75502768051845	-143.13753603762717	-143.95634922638706	-144.56172452318094	-144.0213916576655	-143.97183755864256	-144.62548618740072	-144.303554706773	-144.457200161588	-142.75032388114153	-143.9588860300601	-143.60793137766845	-143.96541925540066	-145.60275215827292	-145.30369505752046	-145.54294170281284	-143.24221397795657	-142.58188499092537	-143.83920987927814	-143.9040433811007	-142.97167133710565	-143.86497782815283	-144.7791715621915	-143.65457572263935	-143.67040921281378	-144.81847880991089	-143.2212224685429	-143.16496095570983	-142.6987584338337	-143.0856676348602
+18_null	121.01285226135494	121.0492691953639	119.94222429360197	119.93161693490953	119.13224564096332	119.89849375109604	120.32632765339717	119.18643981758912	119.49499568731116	120.04340537434723	118.86060965390077	118.4425705312572	119.69793611197167	119.45703599474918	120.63245058137505	120.70580028937368	119.69582513718642	118.2413379397457	120.50085925603909	121.41371220848522	118.77675825416551	119.93443082848131	119.48042245704679	119.92436346240498	119.24914981027973	119.6883593290977	118.78611058030725	119.14984545705057	119.68575006717111	120.23153262587947	118.94991139312131	119.14877948968932	118.0770282450127	119.59911918852423	119.28769336780877	118.92163545107495
 16_null	17.40220683019087	15.12102402792334	15.466362955796416	14.788080523883265	13.412098185923815	14.46000287607303	15.685075306039316	17.168713737308842	15.087509554116401	14.636146633988012	14.463640865638807	16.362978525579106	14.915324162012201	17.845263395194817	15.279160711866703	16.10661135797048	15.301277233618084	16.681140452648318	16.601072409164335	13.955075679385903	15.236847701962233	15.452694616379736	14.580003673868925	13.41965860764166	14.159309917124592	16.044740298695363	15.221300436102306	14.980053998099764	16.191683587944404	14.183077388881639	15.087958606067735	16.257878721763483	15.794936559213765	15.179069333369577	16.431940681259842	14.937881682471762
 17_null	78.43746325635523	77.86872892854892	77.64834489975976	78.49040781126462	77.98371425759402	79.39774565382962	80.86844232526059	80.02994496877774	79.16463654714427	79.28585307378057	77.50522825924493	77.85627727700832	77.98947447938723	78.5804426869695	77.68076847074718	79.43104274248454	76.88699982548832	79.6678695143042	78.85081245977847	78.05021806341595	81.6037181470155	80.2375920072941	77.88815947279353	77.23412817923032	78.12490502356053	80.29962652700054	78.8387799004812	79.20147512554337	79.04361283968156	78.53909929548708	77.93850898253037	80.31551474457403	78.6608431618127	78.68171514861983	78.80411042088949	78.76371577859649
 18_null	121.01285226135494	121.0492691953639	119.94222429360197	119.93161693490953	119.13224564096332	119.89849375109604	120.32632765339717	119.18643981758912	119.49499568731116	120.04340537434723	118.86060965390077	118.4425705312572	119.69793611197167	119.45703599474918	120.63245058137505	120.70580028937368	119.69582513718642	118.2413379397457	120.50085925603909	121.41371220848522	118.77675825416551	119.93443082848131	119.48042245704679	119.92436346240498	119.24914981027973	119.6883593290977	118.78611058030725	119.14984545705057	119.68575006717111	120.23153262587947	118.94991139312131	119.14877948968932	118.0770282450127	119.59911918852423	119.28769336780877	118.92163545107495

cypress/integration/comparisons.spec.ts

@@ -38,7 +38,7 @@ describe("Comparisons", () => {
 
     // Wait until computation is complete
     cy.contains("tr>td", "Two-way ANOVA").next().as("Result");
-    cy.get("@Result", { timeout: 15000 }).should("contain.text", "p: 7.5e-4");
+    cy.get("@Result", { timeout: 25000 }).should("contain.text", "p: 1.9e-4");
 
     cy.contains("span", "Number of selected rows")
       .find("input")
@@ -49,7 +49,8 @@ describe("Comparisons", () => {
       ["Two-way ANOVA", "p: 1.000"],
       ["Main Effect Diff", "p: 0.502"],
       ["Phase Difference", "p: 0.998"],
-      ["Amplitude Difference", "p: 0.952"]
+      ["Amplitude Difference", "p: 0.952"],
+      ["JTK", "p: 4.5e-9"]
     ]) {
       cy.contains("tr>td", algorithm).next().should("contain.text", result);
     }

src/server/models/spreadsheets/spreadsheet.py

@@ -577,16 +577,14 @@ def join_spreadsheets(spreadsheets):
                 df = spreadsheet.df.set_index(index)
                 df = df[~index.duplicated()]
                 index_to_rows = index_to_rows[~index.duplicated()]
-                df = spreadsheet.df.set_index(index)
                 dfs.append(df)
+                rows_list.append(index_to_rows)
 
                 if combined_index is None:
                     combined_index = df.index
                 else:
                     combined_index = combined_index.intersection(df.index)
 
-                rows_list.append(index_to_rows)
-
             # Select only the parts of the data in common to all
             dfs = [df.loc[combined_index] for df in dfs]
             row_numbers = [rows.loc[combined_index] for rows in rows_list]

src/server/static/json/gallery_shares.json

@@ -6,7 +6,7 @@
         "cite": "https://pubmed.ncbi.nlm.nih.gov/26843191/",
         "entries": [
             {"name": "WT vs Post-natal KO",
-             "share": "https://nitecap.org/spreadsheets/share/USM0R6poCiQ3AQUWiE42AvrJhkt9XtwX1c_f7rtb-BF9tHRIIJgvg9F9IpzeLXrMl6HieYqrxX5MFt4dRz5q8oVdh3T32D0yew9o",
+             "share": "https://nitecap.org/spreadsheets/share/g3dj9B--vOTBxhj3AEz4jGpehmiheihM9DVQldwVWZb-PCheyla904D-JSzfzeuX_t-BUSLqMvGcdlUN15LYv9W5ZvVKtpMC9Uj6",
              "description": "Wild-type and post-natal knockout mice (dark-dark)"
              },
             {"name": "WT vs Pre-natal KO",
@@ -45,6 +45,9 @@
             "description": "Heart"},
             {"name": "Liver",
             "share":"https://nitecap.org/spreadsheets/share/QhVo_vwn6ePvj6L64vRDHouIrxdCii6DjvrykODNU3BSnIGr7RHH4-AkyO5gM8ld7flEcallw0l5cbma0PqSR0RD3WaXXWkwdvEI",
+            "description":"Liver"},
+            {"name": "Liver Metabolomics",
+            "share":"https://nitecap.org/spreadsheets/share/YByn9TYjOjtrtO4FzJmgoEyQuBgQ0gfKEIHJ7-qact_QyYlEj6Wj4AiToRg-2TxyQ70RL29wfShP1bwbspm-QKDpU2IMTBSAmyru",
             "description":"Liver"}
         ]
     },

src/server/templates/about.html

@@ -15,6 +15,7 @@ <h1>About</h1>
         Nitecap is an exploratory circadian data analysis tool aimed at high-throughput omics data.
         It provides a web-interface to examine your dataset with highly-responsive visualizations, allowing the easy viewing of hundreds or thousands of genes or other features.
         Nitecap computes some standard rhythmicity tests for you, including JTK, ARSER, Lomb-Scargle, Cosinor, RAIN, and several tests to compare differential rhythmicity across two conditions (such as phase differences).
+        See our <a href="user_guide">User Guide</a> for more help getting started.
 </p>
 
 <p>Nitecap is developed at the <a href="http://bioinf.itmat.upenn.edu/">Institute for Translational Medicine and Therapeutics Bioinformatics, University of Pennsylvania</a> by:</p>

src/server/templates/spreadsheets/comparison.html

@@ -1652,7 +1652,7 @@ <h5 class="modal-title text-white">Compare this spreadsheet</h5>
 
                 // Load the values
                 let spreadsheet = app.spreadsheets_by_id[analysis.spreadsheetId];
-                let reorder = function (list) { return list.map( (x,i) => list[spreadsheet.row_numbers[i]] ); }
+                let reorder = function(list) { return spreadsheet.row_numbers.map( (idx) => list[idx] ); };
                 console.log("Obtained results of ", analysis.name, " for spreadsheet ", analysis.spreadsheetId);
                 if(analysis.name == "Cosinor") {
                     spreadsheet.cosinor_p = Object.freeze(reorder(analysis.p));

src/server/templates/user_guide.html

@@ -210,6 +210,17 @@ <h3 id="slider-bar"> Slider Bar </h3>
             Se the cutoff to 0.1, and you will be given the top gene for which the q-value is less than 0.1.
         </p>
 
+        <h2 id="pvaluesnote">Note on p-values</h2>
+
+        <p>
+            Users of Nitecap should be cautioned against <q>p-hacking</q>.
+            Nitecap presents many different methods producing p-values testing rhythmicity or differential rhythmicity and makes re-running downstream analyses quick.
+            This opens the door to the investigator making choices strategically (or subconciously) to get the desired results regardless of what the data shows.
+            Investigators should be cautious about using any results that depend highly upon the chosen method or cutoff.
+            The most reliable results are often those that give similar results under different choices, and investigators should use the multiple analyses methods provided by Nitecap to check for consistency.
+            Note that since all computed p-values for a specific gene or other feature are computed from the same set of underlying data, they should not be considered as independent.
+        </p>
+
         <h2 id="filters"> Filters </h2>
 
         <p>
@@ -516,6 +527,7 @@ <h3 id="comparison-plots">Comparison Plots</h3>
                         <li> <a href="#profile-plot">Profile Plot</a> </li>
                         <li> <a href="#slider-bar">Slider Bar</a> </li>
                     </ul>
+                    <li> <a href="#pvaluesnote">Note on p-values</a> </li>
                     <li> <a href="#filters">Filters</a> </li>
                     <li> <a href="#pathways">Pathways</a> </li>
                     <li> <a href="#heatmap">Heatmap</a> </li>