Merge branch '0.13.x' into central-develop

Conflicts:
	kalite/distributed/management/commands/setup.py

kalite/coachreports/templates/coachreports/partials/_tabular_exercises_report.html

@@ -7,7 +7,7 @@
             <tr>
                 {% for exercise in exercises %}
                     <th class="headrow data">
-                        <div><a href="{{ exercise.path }}" title='"{% trans exercise.display_name %}"{% if exercise.description %} ({% trans exercise.description %}){% endif %}'>{% trans exercise.title %}</a></div>
+                        <div><a href="{% url 'learn' %}{{ exercise.path }}" title='"{% trans exercise.display_name %}"{% if exercise.description %} ({% trans exercise.description %}){% endif %}'>{% trans exercise.title %}</a></div>
                     </th>
                 {% endfor %}
             </tr>

kalite/coachreports/templates/coachreports/partials/_tabular_videos_report.html

@@ -7,7 +7,7 @@
             <tr>
                 {% for video in videos %}
                     <th class="headrow data">
-                        <div><a href="{{ video.path }}"><span title='"{% trans video.title %}"{% if video.description %} ({% trans video.description %}){% endif %}'>{% trans video.title %}</span></a></div>
+                        <div><a href="{% url 'learn' %}{{ video.path }}"><span title='"{% trans video.title %}"{% if video.description %} ({% trans video.description %}){% endif %}'>{% trans video.title %}</span></a></div>
                     </th>
                 {% endfor %}
             </tr>

kalite/coachreports/templates/coachreports/tabular_view.html

@@ -127,7 +127,7 @@
             {% endif %}
         {% endif %}
         <div class="selection pull-left">
-            <button class="btn btn-primary reports-go-btn" id="display-topic-report">Go</button>
+            <button class="btn btn-primary reports-go-btn" id="display-topic-report">{% trans "Go" %}</button>
         </div>
     </div>
 

kalite/contentload/management/commands/generate_assessment_zips.py

@@ -22,6 +22,9 @@
 
 IMAGE_URL_REGEX = re.compile('https?://[\w\.\-\/]+\/(?P<filename>[\w\.\-]+\.(png|gif|jpg|jpeg))', flags=re.IGNORECASE)
 
+WEB_GRAPHIE_URL_REGEX = re.compile('web\+graphie://ka\-perseus\-graphie\.s3\.amazonaws\.com\/(?P<filename>\w+)', flags=re.IGNORECASE)
+
+
 # this ugly regex looks for links to content on the KA site, also including the markdown link text and surrounding bold markers (*), e.g.
 # **[Read this essay to review](https://www.khanacademy.org/humanities/art-history/art-history-400-1300-medieval---byzantine-eras/anglo-saxon-england/a/the-lindisfarne-gospels)**
 # TODO(jamalex): answer any questions people might have when this breaks!
@@ -38,23 +41,26 @@ def handle_noargs(self, **options):
         # load the assessmentitems
         assessment_items = json.load(open(ASSESSMENT_ITEMS_PATH))
 
-        image_urls = all_image_urls(assessment_items)
+        image_urls = find_all_image_urls(assessment_items)
+        graphie_urls = find_all_graphie_urls(assessment_items)
 
-        logging.info("rewriting image urls")
+        logging.info("rewriting urls")
         new_assessment_items = localize_all_image_urls(assessment_items)
         new_assessment_items = localize_all_content_links(new_assessment_items)
+        new_assessment_items = localize_all_graphie_urls(new_assessment_items)
 
         # TODO(jamalex): We should migrate this away from direct-to-zip so that we can re-run it
         # without redownloading all files. Not possible currently because ZipFile has no `delete`.
         logging.info("downloading images")
         with open(ZIP_FILE_PATH, "w") as f:
             zf = zipfile.ZipFile(f, "w")  # zipfile.ZipFile isn't a context manager yet for python 2.6
             write_assessment_to_zip(zf, new_assessment_items)
-            zip_file_path = download_urls(zf, image_urls)
+            download_urls_to_zip(zf, image_urls)
+            download_urls_to_zip(zf, graphie_urls)
             write_assessment_item_version_to_zip(zf)
             zf.close()
 
-        logging.info("Zip File with images placed in %s" % zip_file_path)
+        logging.info("Zip File with images placed in %s" % ZIP_FILE_PATH)
 
 
 def write_assessment_item_version_to_zip(zf, versionnumber=version.SHORTVERSION):
@@ -66,16 +72,14 @@ def write_assessment_to_zip(zf, assessment_items):
     zf.writestr("assessmentitems.json", assessment_json_string)
 
 
-def download_urls(zf, urls):
+def download_urls_to_zip(zf, urls):
 
     urls = set(urls)
 
     pool = ThreadPool(10)
     download_to_zip_func = lambda url: download_url_to_zip(zf, url)
     pool.map(download_to_zip_func, urls)
 
-    return ZIP_FILE_PATH
-
 
 def download_url_to_zip(zf, url):
     url = url.replace("https://ka-", "http://ka-")
@@ -112,7 +116,7 @@ def fetch_file_from_url_or_cache(url):
     return out
 
 
-def all_image_urls(items):
+def find_all_image_urls(items):
 
     for v in items.itervalues():
         for match in re.finditer(IMAGE_URL_REGEX, v["item_data"]):
@@ -134,14 +138,43 @@ def localize_image_urls(item_data):
     return re.sub(IMAGE_URL_REGEX, _old_image_url_to_content_url, item_data)
 
 
+def find_all_graphie_urls(items):
+
+    for v in items.itervalues():
+        for match in re.finditer(WEB_GRAPHIE_URL_REGEX, v["item_data"]):
+            base_filename = str(match.group(0)).replace("web+graphie:", "https:") # match.group(0) means get the entire string
+            yield base_filename + ".svg"
+            yield base_filename + "-data.json"
+
+
+def localize_all_graphie_urls(items):
+    # we copy so we make sure we don't modify the items passed in to this function
+    newitems = copy.deepcopy(items)
+
+    for item in newitems.itervalues():
+        item['item_data'] = localize_graphie_urls(item['item_data'])
+
+    return newitems
+
+
+def localize_graphie_urls(item_data):
+
+    return re.sub(WEB_GRAPHIE_URL_REGEX, _old_graphie_url_to_content_url, item_data)
+
+
 def convert_urls(item_data):
     """Convert urls in i18n strings into localhost urls.
     This function is used by ka-lite-central/centralserver/i18n/management/commands/update_language_packs.py"""
     item_data = localize_image_urls(item_data)
     item_data = localize_content_links(item_data)
+    item_data = localize_graphie_urls(item_data)
     return item_data
 
 
+def _old_graphie_url_to_content_url(matchobj):
+    return "web+graphie:/content/khan/%s" % matchobj.group("filename")
+
+
 def _old_image_url_to_content_url(matchobj):
     return "/content/khan/%s" % matchobj.group("filename")
 

kalite/contentload/tests/fixtures/assessment_items_sample.json

@@ -5,7 +5,7 @@
             "allytobias"
         ],
         "id": "x0035afac234d8dda",
-        "item_data": "{\"question\":{\"content\":\"The total area is [[\u2603 input-number 1]] $\\\\text{units}^2.$\\n\\n\\n\\n![](https://ka-perseus-graphie.s3.amazonaws.com/8ea5af1fa5a5e8b8e727c3211083111897d23f5d.png)\",\"images\":{\"https://ka-perseus-graphie.s3.amazonaws.com/8ea5af1fa5a5e8b8e727c3211083111897d23f5d.png\":{\"width\":400,\"height\":366}},\"widgets\":{\"input-number 1\":{\"type\":\"input-number\",\"graded\":true,\"options\":{\"value\":40,\"simplify\":\"required\",\"size\":\"normal\",\"inexact\":false,\"maxError\":0.1,\"answerType\":\"number\"}}}},\"answerArea\":{\"type\":\"multiple\",\"options\":{\"content\":\"Fill in the blank.\",\"images\":{},\"widgets\":{}},\"calculator\":false},\"hints\":[{\"content\":\"Let's find the areas of the rectangle and square. Then let's add the areas together. \\n\\nThe area  of the rectangle on the top is  the product of the height and the  width.  We see that the width is $\\\\green 5$ and the height is $\\\\purple 3$.  Therefore\\n\\n$\\\\begin{align} {\\\\qquad}{\\\\text{area of rectangle}} &=\\\\purple3\\\\cdot\\\\green 5 \\\\\\\\\\\\\\\\\\n &= \\\\purple{15}.\\\\\\\\\\\\\\\\\\n\\\\end{align}$\",\"images\":{},\"widgets\":{}},{\"content\":\"The area of a square is the length of its side squared. Matthew's square has a side of length $\\\\blue5$, so we can calculate the area:\\n\\n$\\\\begin{align} {\\\\qquad}\\\\text{area of square} &= \\\\blue5^2 \\\\\\\\\\n &= \\\\green{25}.\\\\end{align}$ \",\"images\":{},\"widgets\":{}},{\"content\":\"Let's add the areas of the triangle and square together to get the total area: \\n\\n$\\\\begin{align} {\\\\qquad}A &= \\\\purple{15}+\\\\green{25} \\\\\\\\\\n &= 40\\\\text{ units}^2.\\\\end{align}$ \\n\\nYou could also compute the area by noticing that the large rectangle has height $\\\\purple 3+\\\\green 5=8$ and width $\\\\green 5$, so the area is $8\\\\times\\\\green 5=40$!\",\"images\":{},\"widgets\":{}},{\"content\":\"Matthew colored in $40\\\\text{ units}^2$. [Watch video](https://www.khanacademy.org/math/early-math/cc-early-math-add-sub-topic/basic-addition-subtraction/v/addition-introduction)\",\"images\":{},\"widgets\":{}}]}",
+        "item_data": "{\"question\":{\"content\":\"The total area is [[\u2603 input-number 1]] $\\\\text{units}^2.$\\n\\n\\n\\n![](https://ka-perseus-graphie.s3.amazonaws.com/8ea5af1fa5a5e8b8e727c3211083111897d23f5d.png)\",\"images\":{\"web+graphie://ka-perseus-graphie.s3.amazonaws.com/9135a0b3072fe7bfdd02d8cf426bfc172c9446a3\":{\"width\":460,\"height\":80},\"https://ka-perseus-graphie.s3.amazonaws.com/8ea5af1fa5a5e8b8e727c3211083111897d23f5d.png\":{\"width\":400,\"height\":366}},\"widgets\":{\"input-number 1\":{\"type\":\"input-number\",\"graded\":true,\"options\":{\"value\":40,\"simplify\":\"required\",\"size\":\"normal\",\"inexact\":false,\"maxError\":0.1,\"answerType\":\"number\"}}}},\"answerArea\":{\"type\":\"multiple\",\"options\":{\"content\":\"Fill in the blank.\",\"images\":{},\"widgets\":{}},\"calculator\":false},\"hints\":[{\"content\":\"Let's find the areas of the rectangle and square. Then let's add the areas together. \\n\\nThe area  of the rectangle on the top is  the product of the height and the  width.  We see that the width is $\\\\green 5$ and the height is $\\\\purple 3$.  Therefore\\n\\n$\\\\begin{align} {\\\\qquad}{\\\\text{area of rectangle}} &=\\\\purple3\\\\cdot\\\\green 5 \\\\\\\\\\\\\\\\\\n &= \\\\purple{15}.\\\\\\\\\\\\\\\\\\n\\\\end{align}$\",\"images\":{},\"widgets\":{}},{\"content\":\"The area of a square is the length of its side squared. Matthew's square has a side of length $\\\\blue5$, so we can calculate the area:\\n\\n$\\\\begin{align} {\\\\qquad}\\\\text{area of square} &= \\\\blue5^2 \\\\\\\\\\n &= \\\\green{25}.\\\\end{align}$ \",\"images\":{},\"widgets\":{}},{\"content\":\"Let's add the areas of the triangle and square together to get the total area: \\n\\n$\\\\begin{align} {\\\\qquad}A &= \\\\purple{15}+\\\\green{25} \\\\\\\\\\n &= 40\\\\text{ units}^2.\\\\end{align}$ \\n\\nYou could also compute the area by noticing that the large rectangle has height $\\\\purple 3+\\\\green 5=8$ and width $\\\\green 5$, so the area is $8\\\\times\\\\green 5=40$!\",\"images\":{},\"widgets\":{}},{\"content\":\"Matthew colored in $40\\\\text{ units}^2$. [Watch video](https://www.khanacademy.org/math/early-math/cc-early-math-add-sub-topic/basic-addition-subtraction/v/addition-introduction)\",\"images\":{},\"widgets\":{}}]}",
         "kind": "AssessmentItem",
         "name": "",
         "parent_id": null,

kalite/contentload/tests/generate_assessment_zips.py

@@ -51,7 +51,7 @@ def test_localize_all_image_urls(self):
         old_item_data = self.assessment_items.values()[0]["item_data"]
         new_item_data = new_items.values()[0]["item_data"]
         self.assertEqual(old_item_data.replace("https://ka-perseus-graphie.s3.amazonaws.com/", "/content/khan/"), new_item_data)
-        self.assertNotIn("amazonaws.com", new_item_data)
+        self.assertNotIn("https://ka-perseus", new_item_data)
 
     def test_localize_all_content_links(self):
         new_items = mod.localize_all_content_links(self.assessment_items)
@@ -62,6 +62,15 @@ def test_localize_all_content_links(self):
             "/learn/khan/math/early-math/cc-early-math-add-sub-topic/basic-addition-subtraction/addition-introduction/"), new_item_data)
         self.assertNotIn("khanacademy.org", new_item_data)
 
+    def test_localize_all_graphie_urls(self):
+        new_items = mod.localize_all_graphie_urls(self.assessment_items)
+        old_item_data = self.assessment_items.values()[0]["item_data"]
+        new_item_data = new_items.values()[0]["item_data"]
+        self.assertEqual(old_item_data.replace(
+            "web+graphie://ka-perseus-graphie.s3.amazonaws.com/",
+            "web+graphie:/content/khan/"), new_item_data)
+        self.assertNotIn("web+graphie://ka-perseus", new_item_data)
+
 
 class TestUrlConversion(TestCase):
 
@@ -97,7 +106,8 @@ def test_command(self, get_method):
         with open(ASSESSMENT_ITEMS_SAMPLE_PATH) as f:
             assessment_items_content = f.read()
 
-        image_requests = len(set(mod.all_image_urls(json.loads(assessment_items_content))))
+        image_requests = len(set(list(mod.find_all_image_urls(json.loads(assessment_items_content)))
+                               + list(mod.find_all_graphie_urls(json.loads(assessment_items_content)))))
 
         get_method.return_value = MagicMock(content=assessment_items_content)
 
@@ -118,6 +128,10 @@ def test_command(self, get_method):
                     continue
                 elif filename.lower().endswith(".png"):
                     continue
+                elif filename.lower().endswith(".svg"):
+                    continue
+                elif filename.lower().endswith("-data.json"):
+                    continue
                 elif filename in ["assessmentitems.json", "assessmentitems.json.version"]:
                     continue
                 else:
@@ -174,7 +188,7 @@ def test_fetch_file_from_url_or_cache(self, get_method):
 
     def test_gets_images_urls_inside_item_data(self):
 
-        result = list(mod.all_image_urls(self.assessment_sample))
+        result = list(mod.find_all_image_urls(self.assessment_sample))
         self.assertIn(
             self.imgurl,
             result,
@@ -184,7 +198,7 @@ def test_gets_images_urls_inside_item_data(self):
     def test_localize_all_image_urls_replaces_with_local_urls(self):
         new_assessment_items = mod.localize_all_image_urls(self.assessment_sample)
 
-        all_images = list(mod.all_image_urls(new_assessment_items))
+        all_images = list(mod.find_all_image_urls(new_assessment_items))
         self.assertNotIn(self.imgurl, all_images)
 
     @patch.object(requests, "get")
@@ -207,7 +221,7 @@ def test_download_url_downloads_all_urls(self, download_method, zipfile_class):
         urls = ["http://test1.com", "http://test2.com"]
         with open(mod.ZIP_FILE_PATH, "w") as f:
             zf = zipfile.ZipFile(f, "w")
-            mod.download_urls(zf, urls)
+            mod.download_urls_to_zip(zf, urls)
             zf.close()
 
         self.assertEqual(download_method.call_count, len(urls))

kalite/control_panel/hbtemplates/data_export/data-export-container.handlebars

@@ -5,10 +5,12 @@
         <div class="col-xs-8">
             <select class="form-control" id="resource-id">
                 <option value="facility_user">{{_ "Facility User Logs" }}</option>
-                <option value="test_log">{{_ "Test Logs" }}</option>
                 <option value="attempt_log">{{_ "Attempt Logs" }}</option>
                 <option value="exercise_log">{{_ "Exercise Logs" }}</option>
+                {{#if is_nalanda}}
                 <option value="store_log">{{_ "Store Transaction Logs" }}</option>
+                <option value="test_log">{{_ "Test Logs" }}</option>
+                {{/if}}
                 {{#if is_central}}
                 <option value="device_log">{{_ "Device Logs" }}</option>
                 {{/if}}

kalite/control_panel/static/js/control_panel/data_export.js

@@ -358,93 +358,3 @@ window.GroupSelectView = Backbone.View.extend({
         }
     }
 });
-
-var DataExportView = Backbone.View.extend({
-    // the containing view
-    template: HB.template('data_export/data-export-container'),
-
-    initialize: function() {
-
-        this.zone_select_view = new ZoneSelectView({
-            org_id: this.options.org_id,
-            model: this.model
-        });
-
-        this.facility_select_view  = new FacilitySelectView({
-            model: this.model
-        });
-
-        this.group_select_view = new GroupSelectView({
-            model: this.model
-        });
-
-        this.render();
-    },
-
-    events: {
-        "click #export-button": "export_data",
-        "change #resource-id": "resource_changed"
-    },
-
-    render: function() {
-        // render container
-        this.$el.html(this.template(this.model.attributes));
-
-        // append zone, facility & group select views.
-        this.$('#student-select-container').append(this.zone_select_view.$el);
-        this.$('#student-select-container').append(this.facility_select_view.$el);
-        this.$('#student-select-container').append(this.group_select_view.$el);
-    },
-
-    resource_changed: function() {
-        this.model.set({
-            resource_id: this.$('#resource-id').val()
-        });
-    },
-
-    resource_endpoint: function() {
-        // Return the API url endpoint for the current resource id
-        var resource_id = this.model.get("resource_id");
-        switch  (resource_id) {
-            case "facility_user":
-                return FACILITY_USER_CSV_URL;
-            case "test_log":
-                return TEST_LOG_CSV_URL;
-            case "attempt_log":
-                return ATTEMPT_LOG_CSV_URL;
-            case "exercise_log":
-                return EXERCISE_LOG_CSV_URL;
-            case "device_log":
-                return DEVICE_LOG_CSV_URL;
-        }
-    },
-
-    export_data: function(ev) {
-        ev.preventDefault();
-
-        // Update export link based on currently selected paramters
-        var zone_id = this.model.get("zone_id");
-        var facility_id = this.model.get("facility_id");
-        var group_id = this.model.get("group_id");
-        var resource_endpoint = this.resource_endpoint();
-
-        // If no zone_id, all is selected, so compile a comma seperated string
-        // of zone ids to pass to endpoint
-        var zone_ids = "";
-        if (zone_id===undefined || zone_id==="") {
-            zone_ids = _.map(this.zone_select_view.zone_list.models, function(zone) { return zone.get("id"); }).join();
-        }
-
-        var export_params = "?" + $.param({
-            group_id: group_id,
-            facility_id: facility_id,
-            zone_id: zone_id,
-            zone_ids: zone_ids,
-            format:"csv",
-            limit:0
-        });
-
-        var export_data_url = this.resource_endpoint() + export_params;
-        window.location = export_data_url;
-    }
-});

kalite/control_panel/templates/control_panel/data_export.html

@@ -35,7 +35,8 @@
                     facility_id: "{{ facility_id }}", 
                     // b/c we hardcode in template, initialize this for state model
                     resource_id: "facility_user",  
-                    is_central: {{ is_central|lower }}
+                    is_central: {{ is_central|lower }},
+                    is_nalanda: ds.ab_testing.is_config_package_nalanda
                 }),
                 org_id: "{{ org_id }}"
             });

kalite/control_panel/templates/control_panel/facility_management.html

@@ -4,7 +4,7 @@
 {% load staticfiles %}
 
 {% block i18n_do_not_translate %}
-    {% block control_panel_active %}{% endblock %}
+    {% block control_panel_active %}active{% endblock %}
     {% block user_active %}active{% endblock user_active %}
 {% endblock i18n_do_not_translate %}
 

kalite/distributed/demo_middleware.py

@@ -5,6 +5,8 @@
 * Links to documentation on how to use KA Lite
 * Prevents certain sensitive resources from being accessed (like the admin interface)
 """
+import re
+
 from django.conf import settings
 from django.contrib import messages
 from django.core.exceptions import PermissionDenied
@@ -14,7 +16,23 @@
 
 
 def is_static_file(path):
-    return path.startswith(settings.STATIC_URL) or path.startswith(settings.MEDIA_URL)
+
+    url_exceptions = [
+        "^/admin/*",
+        "^/api/*",
+        "^{url}/*".format(url=settings.STATIC_URL),
+        "^/data/*",
+        "^{url}/*".format(url=settings.MEDIA_URL),
+        "^/handlebars/*",
+        "^.*/_generated/*"
+    ]
+
+    for item in url_exceptions:
+        p = re.compile(item)
+        if p.match(path):
+            return True
+
+    return False
 
 class LinkUserManual:
     """Shows a message with a link to the user's manual, from the homepage."""