diff --git a/contentcuration/contentcuration/frontend/shared/leUtils/Languages.js b/contentcuration/contentcuration/frontend/shared/leUtils/Languages.js index 32365b7132..7c95090633 100644 --- a/contentcuration/contentcuration/frontend/shared/leUtils/Languages.js +++ b/contentcuration/contentcuration/frontend/shared/leUtils/Languages.js @@ -184,7 +184,7 @@ const LanguagesMap = new Map([ lang_subcode: null, readable_name: 'Brahui', native_name: 'Brahui', - lang_direction: 'ltr', + lang_direction: 'rtl', }, ], [ @@ -1130,7 +1130,7 @@ const LanguagesMap = new Map([ lang_subcode: null, readable_name: 'Northern Pashto', native_name: 'Northern Pashto', - lang_direction: 'ltr', + lang_direction: 'rtl', }, ], [ @@ -1471,7 +1471,7 @@ const LanguagesMap = new Map([ lang_subcode: null, readable_name: 'Southern Balochi', native_name: 'Southern Balochi', - lang_direction: 'ltr', + lang_direction: 'rtl', }, ], [ @@ -1626,7 +1626,7 @@ const LanguagesMap = new Map([ lang_subcode: null, readable_name: 'Uighur; Uyghur', native_name: 'Uy\u01a3urq\u0259, \u0626\u06c7\u064a\u063a\u06c7\u0631\u0686\u06d5\u200e', - lang_direction: 'ltr', + lang_direction: 'rtl', }, ], [ @@ -2540,6 +2540,17 @@ const LanguagesMap = new Map([ lang_direction: 'rtl', }, ], + [ + 'prs', + { + id: 'prs', + lang_code: 'prs', + lang_subcode: null, + readable_name: 'Dari', + native_name: '\u062f\u0631\u06cc', + lang_direction: 'rtl', + }, + ], [ 'arq', { @@ -2604,7 +2615,7 @@ const LanguagesMap = new Map([ readable_name: 'Kashmiri', native_name: '\u0915\u0936\u094d\u092e\u0940\u0930\u0940, \u0643\u0634\u0645\u064a\u0631\u064a\u200e', - lang_direction: 'ltr', + lang_direction: 'rtl', }, ], [ @@ -2740,7 +2751,7 @@ const LanguagesMap = new Map([ readable_name: 'Sindhi', native_name: '\u0938\u093f\u0928\u094d\u0927\u0940, \u0633\u0646\u068c\u064a\u060c \u0633\u0646\u062f\u06be\u06cc\u200e', - lang_direction: 'ltr', + lang_direction: 'rtl', }, ], [ @@ -2784,7 +2795,7 @@ const LanguagesMap = new Map([ lang_subcode: null, readable_name: 'Punjabi', native_name: '\u0a2a\u0a70\u0a1c\u0a3e\u0a2c\u0a40', - lang_direction: 'ltr', + lang_direction: 'rtl', }, ], [ @@ -3390,6 +3401,7 @@ export const LanguagesNames = { HE: 'he', UR: 'ur', AR: 'ar', + PRS: 'prs', ARQ: 'arq', FA: 'fa', PS: 'ps', diff --git a/contentcuration/contentcuration/frontend/shared/leUtils/MasteryModels.js b/contentcuration/contentcuration/frontend/shared/leUtils/MasteryModels.js index 57240f5c47..a909b5eefb 100644 --- a/contentcuration/contentcuration/frontend/shared/leUtils/MasteryModels.js +++ b/contentcuration/contentcuration/frontend/shared/leUtils/MasteryModels.js @@ -2,6 +2,7 @@ const MasteryModels = new Set([ 'do_all', 'm_of_n', + 'pre_post_test', 'num_correct_in_a_row_2', 'num_correct_in_a_row_3', 'num_correct_in_a_row_5', @@ -15,6 +16,7 @@ export const MasteryModelsList = Array.from(MasteryModels); export const MasteryModelsNames = { DO_ALL: 'do_all', M_OF_N: 'm_of_n', + PRE_POST_TEST: 'pre_post_test', NUM_CORRECT_IN_A_ROW_2: 'num_correct_in_a_row_2', NUM_CORRECT_IN_A_ROW_3: 'num_correct_in_a_row_3', NUM_CORRECT_IN_A_ROW_5: 'num_correct_in_a_row_5', diff --git a/contentcuration/contentcuration/frontend/shared/views/LanguageDropdown.vue b/contentcuration/contentcuration/frontend/shared/views/LanguageDropdown.vue index ddf39fdfee..2ab041dfa6 100644 --- a/contentcuration/contentcuration/frontend/shared/views/LanguageDropdown.vue +++ b/contentcuration/contentcuration/frontend/shared/views/LanguageDropdown.vue @@ -113,6 +113,13 @@ }, methods: { languageText(item) { + // VAutocomplete eagerly evaluates getText(internalValue) as a fallback arg to + // getValue, even when that fallback isn't needed. In multiple mode, internalValue + // is an Array, so languageText receives the array directly. Return early to avoid + // calling .split() on undefined. + if (Array.isArray(item)) { + return ''; + } const firstNativeName = item.native_name.split(',')[0].trim(); return this.$tr('languageItemText', { language: firstNativeName, code: item.id }); }, diff --git a/contentcuration/contentcuration/frontend/shared/views/__tests__/languageDropdown.spec.js b/contentcuration/contentcuration/frontend/shared/views/__tests__/languageDropdown.spec.js index 8bdcd165fd..13244d39b2 100644 --- a/contentcuration/contentcuration/frontend/shared/views/__tests__/languageDropdown.spec.js +++ b/contentcuration/contentcuration/frontend/shared/views/__tests__/languageDropdown.spec.js @@ -81,4 +81,15 @@ describe('languageDropdown', () => { const item = { native_name: '', id: 'de' }; expect(wrapper.vm.languageText(item)).toBe(' (de)'); }); + + it('returns empty string when called with an array (multiple mode VAutocomplete internal call)', () => { + const wrapper = shallowMount(LanguageDropdown, { + mocks: { + $tr: (key, params) => `${params.language} (${params.code})`, + }, + }); + // VAutocomplete eagerly evaluates getText(internalValue) as a fallback to getValue. + // In multiple mode, internalValue is an Array, so languageText receives the array. + expect(wrapper.vm.languageText(['en', 'fr'])).toBe(''); + }); }); diff --git a/contentcuration/contentcuration/management/commands/fix_missing_import_sources.py b/contentcuration/contentcuration/management/commands/fix_missing_import_sources.py index 6f40cb569a..6c5210ef7a 100644 --- a/contentcuration/contentcuration/management/commands/fix_missing_import_sources.py +++ b/contentcuration/contentcuration/management/commands/fix_missing_import_sources.py @@ -1,6 +1,11 @@ import csv +import io import logging import time +import uuid +from pathlib import Path +from typing import Optional +from typing import Tuple from django.core.management.base import BaseCommand from django.db.models import Exists @@ -9,20 +14,101 @@ from django.db.models import Q from django.db.models.expressions import F from django_cte import With +from le_utils.constants import content_kinds from contentcuration.models import Channel from contentcuration.models import ContentNode +from contentcuration.models import License logger = logging.getLogger(__name__) +class LicensingFixesLookup(object): + """Consolidates logic for reading and processing the licensing fixes from the CSV""" + + def __init__(self): + self._lookup = {} + self._license_lookup = {} + + def load(self, fp: io.TextIOWrapper): + """Loads the data from the CSV file, and the necessary license data from the database""" + reader = csv.DictReader(fp) + license_names = set() + + # create a lookup index by channel ID from the CSV data + for row in reader: + lookup_key = f"{uuid.UUID(row['channel_id']).hex}:{row.get('kind', '')}" + self._lookup[lookup_key] = row + if row["license_name"]: + license_names.add(row["license_name"]) + + # load all licenses, regardless of whether they are named in the CSV + license_lookup_by_name = {} + for lic in License.objects.all(): + self._license_lookup[lic.id] = lic + license_lookup_by_name[lic.license_name] = lic + license_names.discard(lic.license_name) + + # ensure we've found all the licenses + if len(license_names): + raise ValueError(f"Could not find all licenses: {license_names}") + + # we now are certain all licenses are found + for info in self._lookup.values(): + if info["license_name"]: + info["license_id"] = license_lookup_by_name[info["license_name"]].id + + def get_info( + self, + channel_id: str, + kind: str, + license_id: Optional[int], + license_description: Optional[str], + copyright_holder: Optional[str], + ) -> Tuple[Optional[int], Optional[str], Optional[str]]: + """ + Determines the complete licensing metadata, given the current metadata, and comparing it + with what would make the node complete. + + :param channel_id: The channel the node was sourced from + :param kind: The content kind of the node + :param license_id: The current license_id of the node + :param license_description: The current license_description of the node + :param copyright_holder: The current copyright_holder of the node + :return: A tuple of (license_id, license_description, copyright_holder) to use on the node + """ + # first check kind-specific metadata, fallback to channel-wide (no kind) + info = self._lookup.get(f"{channel_id}:{kind}", None) + if info is None: + info = self._lookup.get(f"{channel_id}:", None) + + if info is None: + logger.warning(f"Failed to find licensing info for channel: {channel_id}") + return license_id, license_description, copyright_holder + + if not license_id: + license_id = info["license_id"] + + if not license_id: + return None, license_description, copyright_holder + + license_obj = self._license_lookup.get(license_id) + + if license_obj.is_custom and not license_description: + license_description = info["license_description"] + + if license_obj.copyright_holder_required and not copyright_holder: + copyright_holder = info["copyright_holder"] + + return license_id, license_description, copyright_holder + + class Command(BaseCommand): """ Audits nodes that have imported content from public channels and whether the imported content - has a missing source node. - - TODO: this does not yet FIX them + has a missing source node. We've determined that pretty much all of these have incomplete + licensing data """ def handle(self, *args, **options): @@ -71,32 +157,27 @@ def handle(self, *args, **options): logger.info("=== Iterating over private destination channels. ===") channel_count = 0 - total_node_count = 0 - - with open("fix_missing_import_sources.csv", "w", newline="") as csv_file: - csv_writer = csv.DictWriter( - csv_file, - fieldnames=[ - "channel_id", - "channel_name", - "contentnode_id", - "contentnode_title", - "public_channel_id", - "public_channel_name", - "public_channel_deleted", - ], - ) - csv_writer.writeheader() + total_fixed = 0 + lookup = LicensingFixesLookup() + + command_dir = Path(__file__).parent + csv_path = command_dir / "licensing_fixes_lookup.csv" + + with csv_path.open("r", encoding="utf-8", newline="") as csv_file: + lookup.load(csv_file) - for channel in destination_channels.iterator(): - node_count = self.handle_channel(csv_writer, channel) + # skip using an iterator here, to limit transaction duration to `handle_channel` + for channel in destination_channels: + node_count = self.handle_channel(lookup, channel) - if node_count > 0: - total_node_count += node_count - channel_count += 1 + if node_count > 0: + total_fixed += node_count + channel_count += 1 logger.info("=== Done iterating over private destination channels. ===") - logger.info(f"Found {total_node_count} nodes across {channel_count} channels.") + logger.info( + f"Fixed incomplete licensing data on {total_fixed} nodes across {channel_count} channels." + ) logger.info(f"Finished in {time.time() - start}") def get_public_cte(self) -> With: @@ -110,7 +191,15 @@ def get_public_cte(self) -> With: name="public_cte", ) - def handle_channel(self, csv_writer: csv.DictWriter, channel: dict) -> int: + def handle_channel(self, lookup: LicensingFixesLookup, channel: dict) -> int: + """ + Goes through the nodes of the channel, that were imported from public channels, but no + longer have a valid source node. For each node, it applies license metadata as necessary + + :param lookup: The lookup utility to pull licensing data from + :param channel: The channel to fix + :return: The total node count that are now marked complete as a result of the fixes + """ public_cte = self.get_public_cte() channel_id = channel["id"] channel_name = channel["name"] @@ -127,6 +216,7 @@ def handle_channel(self, csv_writer: csv.DictWriter, channel: dict) -> int: public_channel_name=public_cte.col.name, public_channel_deleted=public_cte.col.deleted, ) + .exclude(kind=content_kinds.TOPIC) .filter( Q(public_channel_deleted=True) | ~Exists( @@ -136,29 +226,51 @@ def handle_channel(self, csv_writer: csv.DictWriter, channel: dict) -> int: ) ) ) - .values( - "public_channel_id", - "public_channel_name", - "public_channel_deleted", - contentnode_id=F("id"), - contentnode_title=F("title"), - ) ) # Count and log results node_count = missing_source_nodes.count() + processed = 0 + was_complete = 0 + unfixed = 0 + now_complete = 0 - # TODO: this will be replaced with logic to correct the missing source nodes - if node_count > 0: + def _log(): logger.info( - f"{channel_id}:{channel_name}\t{node_count} node(s) with missing source nodes." + f"Fixing {channel_id}:{channel_name}\ttotal: {node_count}; before: {was_complete} unfixed: {unfixed}; after: {now_complete};" ) - row_dict = { - "channel_id": channel_id, - "channel_name": channel_name, - } - for node_dict in missing_source_nodes.iterator(): - row_dict.update(node_dict) - csv_writer.writerow(row_dict) - - return node_count + + if node_count > 0: + for node in missing_source_nodes.iterator(): + # determine the new license metadata + license_id, license_description, copyright_holder = lookup.get_info( + node.original_channel_id, + node.kind_id, + node.license_id, + node.license_description, + node.copyright_holder, + ) + + # if there isn't a license, there's nothing to do + if not license_id: + unfixed += 1 + # cannot fix + continue + + if node.complete: + was_complete += 1 + + # apply updates + node.license_id = license_id + node.license_description = license_description + node.copyright_holder = copyright_holder + if not node.mark_complete(): + now_complete += 1 + node.save() + processed += 1 + if processed % 100 == 0: + _log() + + _log() + + return now_complete - was_complete diff --git a/contentcuration/contentcuration/management/commands/licensing_fixes_lookup.csv b/contentcuration/contentcuration/management/commands/licensing_fixes_lookup.csv new file mode 100644 index 0000000000..c510984037 --- /dev/null +++ b/contentcuration/contentcuration/management/commands/licensing_fixes_lookup.csv @@ -0,0 +1,67 @@ +channel_id,channel_name,kind,license_id,license_name,license_description,copyright_holder +f9d3e0e4-6ea2-5789-bbed-672ff6a399ed,African Storybook Library (multiple languages),,,CC BY,,African Storybook Initiative +d0ef6f71-e4fe-4e54-bb87-d7dab5eeaae2,Be Strong: Internet safety resources,,,CC BY-NC-ND,,Vodafone +2d7b056d-668a-58ee-9244-ccf76108cbdb,Book Dash,,,CC BY,,http://bookdash.org/ +922e9c57-6c2f-59e5-9389-142b136308ff,Career Girls,,,Special Permissions,For use on Kolibri,Career Girls +da53f90b-1be2-5752-a046-82bbc353659f,Ciencia NASA,,,CC BY,,NASA +0294a064-f722-4899-887c-e07bd47f9991,Citoyennes de la Terre,,,CC BY,,Florence Piron +604ad3b8-5d84-4dd8-9ee7-0fa12a9a5a6e,CREE+,,,CC BY-NC-SA,,"Publicado por el Lic. Edelberto Andino(edelberto.andino.ea@gmail.com) para ser utilizado con fines educativos únicamente, no debe ser utilizado con fines lucrativos de ninguna índole." +ef2ead65-de76-4ea4-a27b-ba6df5282c74,CSpathshala - सीएसपाठशाला (हिंदी),,,CC BY,,ए सि एम् इंडिया +7e68bc59-d430-4e71-8a07-50b1b87125ad,Cultura Emprendedora,,,CC BY-NC-SA,"The Attribution-NonCommercial-ShareAlike License lets others remix, tweak, and build upon your work non-commercially, as long as they credit you and license their new creations under the identical terms.",Junta de Andalucia +c51a0f84-2fed-427c-95ac-ff9bb4a21e3c,EENET Inclusive Education Training Materials,,,CC BY-NC-SA,,Enabling Education Network (EENET) +0e173fca-6e90-52f8-a474-a2fb84055faf,Global Digital Library - Book Catalog,,,CC BY,,Enabling Writers Initiative +624e09bb-5eeb-4d20-aa8d-e62e7b4778a0,How to get started with Kolibri,,,CC BY-NC,,Learning Equality +378cf412-8c85-4c27-95c1-00b5aca7a3ed,Inclusive Home Learning Activities,,,CC BY,"The Attribution License lets others distribute, remix, tweak, and build upon your work, even commercially, as long as they credit you for the original creation. This is the most accommodating of licenses offered. Recommended for maximum dissemination and use of licensed materials.",EENET – Enabling Education Network +d76da4d3-6cfd-5927-9b57-5dfc6017aa13,Kamkalima (العربيّة),,,CC BY-NC-ND,,Kamkalima +2fd54ca4-7a8f-59c9-9fce-faaa3894c19e,Khan Academy (English - CBSE India Curriculum),video,,CC BY-NC-SA,,Khan Academy +2fd54ca4-7a8f-59c9-9fce-faaa3894c19e,Khan Academy (English - CBSE India Curriculum),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +c9d7f950-ab6b-5a11-99e3-d6c10d7f0103,Khan Academy (English - US curriculum),video,,CC BY-NC-SA,,Khan Academy +c9d7f950-ab6b-5a11-99e3-d6c10d7f0103,Khan Academy (English - US curriculum),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +c1f2b7e6-ac9f-56a2-bb44-fa7a48b66dce,Khan Academy (Español),video,,CC BY-NC-SA,,Khan Academy +c1f2b7e6-ac9f-56a2-bb44-fa7a48b66dce,Khan Academy (Español),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +878ec2e6-f88c-5c26-8b1b-e6f202833cd4,Khan Academy (Français),video,,CC BY-NC-SA,,Khan Academy +878ec2e6-f88c-5c26-8b1b-e6f202833cd4,Khan Academy (Français),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +801a5f02-9420-5569-8918-edcff6494185,Khan Academy (Italiano),video,,CC BY-NC-SA,,Khan Academy +801a5f02-9420-5569-8918-edcff6494185,Khan Academy (Italiano),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +ec164fee-25ee-5262-96e6-8f7c10b1e169,Khan Academy (Kiswahili),video,,CC BY-NC-SA,,Khan Academy +ec164fee-25ee-5262-96e6-8f7c10b1e169,Khan Academy (Kiswahili),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +2ac071c4-6723-54f2-aa78-953448f81e50,Khan Academy (Português - Brasil),video,,CC BY-NC-SA,,Khan Academy +2ac071c4-6723-54f2-aa78-953448f81e50,Khan Academy (Português - Brasil),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +c3231d84-4f8d-5bb1-b4cb-c6a7ddd91eb7,Khan Academy (Português (Portugal)),video,,CC BY-NC-SA,,Khan Academy +c3231d84-4f8d-5bb1-b4cb-c6a7ddd91eb7,Khan Academy (Português (Portugal)),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +09ee940e-1069-53a2-b671-6e1020a0ce3f,Khan Academy (български език),video,,CC BY-NC-SA,,Khan Academy +09ee940e-1069-53a2-b671-6e1020a0ce3f,Khan Academy (български език),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +a53592c9-72a8-594e-9b69-5aa127493ff6,Khan Academy (हिन्दी),video,,CC BY-NC-SA,,Khan Academy +a53592c9-72a8-594e-9b69-5aa127493ff6,Khan Academy (हिन्दी),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +a03496a6-de09-5e7b-a9d2-4291a487c78d,Khan Academy (বাংলা),video,,CC BY-NC-SA,,Khan Academy +a03496a6-de09-5e7b-a9d2-4291a487c78d,Khan Academy (বাংলা),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +5357e525-81c3-567d-a4f5-6d56badfeac7,Khan Academy (ગુજરાતી),video,,CC BY-NC-SA,,Khan Academy +5357e525-81c3-567d-a4f5-6d56badfeac7,Khan Academy (ગુજરાતી),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +2b608c6f-d4c3-5c34-b738-7e3dd7b53265,Khan Academy (ဗမာစာ),video,,CC BY-NC-SA,,Khan Academy +2b608c6f-d4c3-5c34-b738-7e3dd7b53265,Khan Academy (ဗမာစာ),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +f5b71417-b1f6-57fc-a4d1-aaecd23e4067,Khan Academy (ភាសាខ្មែរ),video,,CC BY-NC-SA,,Khan Academy +f5b71417-b1f6-57fc-a4d1-aaecd23e4067,Khan Academy (ភាសាខ្មែរ),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +ec599e77-f9ad-5802-8975-e8a26e6f1821,Khan Academy (中文(中国)),video,,CC BY-NC-SA,,Khan Academy +ec599e77-f9ad-5802-8975-e8a26e6f1822,Khan Academy (中文(中国)),exercise,,Special Permissions,Permission granted to distribute through Kolibri for non-commercial use,Khan Academy +913efe9f-14c6-5cb1-b234-02f21f056e99,MIT Blossoms,,,CC BY-NC-SA,,MIT Blossoms +fc47aee8-2e01-53e2-a301-97d3fdee1128,Open Stax,,,CC BY,"The Attribution License lets others distribute, remix, tweak, and build upon your work, even commercially, as long as they credit you for the original creation. This is the most accommodating of licenses offered. Recommended for maximum dissemination and use of licensed materials.",Rice University +b8bd7770-063d-40a8-bd9b-30d4703927b5,PBS SoCal: Family Math,,,All Rights Reserved,,PBS SoCal +197934f1-4430-5350-b582-0c7c4dd8e194,PhET Interactive Simulations,,,CC BY,,"PhET Interactive Simulations, University of Colorado Boulder" +aa254505-59b5-5bd7-9bc9-0c09dfb805d2,PhET simulações interativas,,,CC BY,,"PhET Interactive Simulations, University of Colorado Boulder" +889f0c34-b275-507a-b8d3-7d2da2d03aa9,PhET – інтерактивне моделювання,,,CC BY,,"PhET Interactive Simulations, University of Colorado Boulder" +f6cb302e-f659-4db4-b4a0-4b4991a595c2,Plan Educativo TIC Basico,,,CC BY-NC-SA,"The Attribution-NonCommercial-ShareAlike License lets others remix, tweak, and build upon your work non-commercially, as long as they credit you and license their new creations under the identical terms.",Junta de Andalucia +e832106c-6398-54e1-8161-6015a8b87910,PraDigi,,,CC BY-NC-SA,"The Attribution-NonCommercial-ShareAlike License lets others remix, tweak, and build upon your work non-commercially, as long as they credit you and license their new creations under the identical terms.",PraDigi +131e543d-becf-5776-bb13-cfcfddf05605,Pratham Books' StoryWeaver,,,CC BY,"The Attribution License lets others distribute, remix, tweak, and build upon your work, even commercially, as long as they credit you for the original creation. This is the most accommodating of licenses offered. Recommended for maximum dissemination and use of licensed materials.",Pratham Books +f758ac6a-d39c-452f-9566-58da6ad7d3cc,Project Based Learning with Kolibri,,,CC BY,,Learning Equality +305b12ea-5ea8-4fa1-8f93-3705c23f5ee0,School of Thought,,,CC BY,,School of Thought +3e464ee1-2f6a-50a7-81cd-df59147b48b1,Sikana (English),,,CC BY-NC-ND,,Sikana Education +30c71c99-c42c-57d1-81e8-aeafd2e15e5f,Sikana (Español),,,CC BY-NC-ND,"The Attribution-NonCommercial-NoDerivs License is the most restrictive of our six main licenses, only allowing others to download your works and share them with others as long as they credit you, but they can't change them in any way or use them commercially.",Sikana Education +8ef625db-6e86-506c-9a3b-ac891e413fff,Sikana (Français),,,CC BY-NC-ND,"The Attribution-NonCommercial-NoDerivs License is the most restrictive of our six main licenses, only allowing others to download your works and share them with others as long as they credit you, but they can't change them in any way or use them commercially.",Sikana Education +f4715a77-6972-5c72-9d25-d29977b8b308,Similasyon Enteraktif PhET,,,CC BY,,"PhET Interactive Simulations, University of Colorado Boulder" +8fa678af-1dd0-5329-bf32-18c549b84996,Simulaciones interactivas PhET,,,CC BY,,"PhET Interactive Simulations, University of Colorado Boulder" +a9b25ac9-8147-42c8-83ce-1b0579448337,TESSA - Teacher Resources,,,CC BY-NC-SA,,Open University +74f36493-bb47-5b62-935f-a8705ed59fed,Thoughtful Learning,,,CC BY-NC-SA,"The Attribution-NonCommercial-ShareAlike License lets others remix, tweak, and build upon your work non-commercially, as long as they credit you and license their new creations under the identical terms.",Thoughtful Learning +000409f8-1dbe-5d1b-a671-01cb9fed4530,Touchable Earth (en),,,Special Permissions,Permission has been granted by Touchable Earth to distribute this content through Kolibri.,Touchable Earth Foundation (New Zealand) +b336c2e2-c45c-53d5-b24e-5c476a54b077,Touchable Earth (fr),,,Special Permissions,Permission has been granted by Touchable Earth to distribute this content through Kolibri.,Touchable Earth Foundation (New Zealand) +08a53136-a155-5f64-b049-6b3e1318b0cd,Ubongo Kids,,,CC BY-NC-ND,"The Attribution-NonCommercial-NoDerivs License is the most restrictive of our six main licenses, only allowing others to download your works and share them with others as long as they credit you, but they can't change them in any way or use them commercially.",Ubongo Media +237e5975-bce2-5bf6-aff3-98f4c17516f3,,,,,, diff --git a/contentcuration/contentcuration/tests/management/commands/test_fix_missing_import_sources.py b/contentcuration/contentcuration/tests/management/commands/test_fix_missing_import_sources.py index e624313ff8..bdaa7f0c57 100644 --- a/contentcuration/contentcuration/tests/management/commands/test_fix_missing_import_sources.py +++ b/contentcuration/contentcuration/tests/management/commands/test_fix_missing_import_sources.py @@ -1,8 +1,12 @@ -from unittest.mock import mock_open +from pathlib import Path from unittest.mock import patch from django.core.management import call_command +from le_utils.constants import content_kinds +from contentcuration.management.commands.fix_missing_import_sources import ( + LicensingFixesLookup, +) from contentcuration.tests import testdata from contentcuration.tests.base import StudioTestCase @@ -11,22 +15,7 @@ class CommandTestCase(StudioTestCase): """Test suite for the fix_missing_import_sources management command""" def setUp(self): - open_patcher = patch( - "contentcuration.management.commands.fix_missing_import_sources.open", - mock_open(), - ) - self.mock_open = open_patcher.start() - self.mock_file = self.mock_open.return_value - self.mock_file.__enter__.return_value = self.mock_file - self.addCleanup(open_patcher.stop) - - csv_writer_patcher = patch( - "contentcuration.management.commands.fix_missing_import_sources.csv.DictWriter" - ) - self.mock_csv_writer = csv_writer_patcher.start() - self.mock_csv_writer_instance = self.mock_csv_writer.return_value - self.addCleanup(csv_writer_patcher.stop) - + super().setUp() self.public_channel = testdata.channel("Public Channel") self.public_channel.public = True self.public_channel.save() @@ -43,58 +32,189 @@ def setUp(self): target=self.private_channel.main_tree ) - def test_handle__opens_csv_file(self): - call_command("fix_missing_import_sources") - - self.mock_open.assert_called_once_with( - "fix_missing_import_sources.csv", "w", newline="" + def test_handle__uses_lookup_and_applies_fix_for_missing_source(self): + self.original_node.delete() + special_permissions_id = 9 + self.copied_node.refresh_from_db() + self.copied_node.license = None + self.copied_node.license_description = "" + self.copied_node.copyright_holder = "" + self.copied_node.save() + + with patch( + "contentcuration.management.commands.fix_missing_import_sources.LicensingFixesLookup" + ) as lookup_cls: + lookup = lookup_cls.return_value + lookup.get_info.return_value = ( + special_permissions_id, + "Permission granted to distribute through Kolibri for non-commercial use", + "Khan Academy", + ) + + call_command("fix_missing_import_sources") + + lookup_cls.assert_called_once() + lookup.load.assert_called_once() + lookup.get_info.assert_called_once() + + self.copied_node.refresh_from_db() + self.assertEqual(self.copied_node.license_id, special_permissions_id) + self.assertEqual( + self.copied_node.license_description, + "Permission granted to distribute through Kolibri for non-commercial use", ) + self.assertEqual(self.copied_node.copyright_holder, "Khan Academy") - self.mock_csv_writer.assert_called_once_with( - self.mock_file, - fieldnames=[ - "channel_id", - "channel_name", - "contentnode_id", - "contentnode_title", - "public_channel_id", - "public_channel_name", - "public_channel_deleted", - ], + def test_handle__applies_fix_for_deleted_public_channel(self): + cc_by_nc_sa_id = 5 + self.public_channel.deleted = True + self.public_channel.save(actor_id=testdata.user().id) + self.copied_node.license = None + self.copied_node.license_description = "" + self.copied_node.copyright_holder = "" + self.copied_node.save() + + with patch( + "contentcuration.management.commands.fix_missing_import_sources.LicensingFixesLookup" + ) as lookup_cls: + lookup = lookup_cls.return_value + lookup.get_info.return_value = ( + cc_by_nc_sa_id, + "", + "Khan Academy", + ) + + call_command("fix_missing_import_sources") + + lookup.get_info.assert_called_once_with( + self.public_channel.id, "video", None, "", "" ) - self.mock_csv_writer_instance.writeheader.assert_called_once() - self.mock_csv_writer_instance.writerow.assert_not_called() + self.copied_node.refresh_from_db() + self.assertEqual(self.copied_node.license_id, cc_by_nc_sa_id) + self.assertEqual(self.copied_node.license_description, "") + self.assertEqual(self.copied_node.copyright_holder, "Khan Academy") + + def test_handle__skips_node_when_lookup_returns_no_license(self): + self.original_node.delete() + self.copied_node.refresh_from_db() + original_license_id = self.copied_node.license_id + self.copied_node.license_description = "Nothing" + self.copied_node.copyright_holder = "Nothing" + self.copied_node.save() + + with patch( + "contentcuration.management.commands.fix_missing_import_sources.LicensingFixesLookup" + ) as lookup_cls: + lookup = lookup_cls.return_value + lookup.get_info.return_value = (None, "Nothing", "Nothing") + + call_command("fix_missing_import_sources") + + lookup.get_info.assert_called_once() + + self.copied_node.refresh_from_db() + self.assertEqual(self.copied_node.license_id, original_license_id) + self.assertEqual(self.copied_node.license_description, "Nothing") + self.assertEqual(self.copied_node.copyright_holder, "Nothing") - def test_handle__finds_missing(self): + def test_handle__skips_topic_nodes_with_missing_source(self): self.original_node.delete() - call_command("fix_missing_import_sources") - - self.mock_csv_writer_instance.writerow.assert_called_once_with( - { - "channel_id": self.private_channel.id, - "channel_name": self.private_channel.name, - "contentnode_id": self.copied_node.id, - "contentnode_title": self.copied_node.title, - "public_channel_id": self.public_channel.id, - "public_channel_name": self.public_channel.name, - "public_channel_deleted": False, - } + self.copied_node.refresh_from_db() + self.copied_node.kind_id = content_kinds.TOPIC + self.copied_node.license = None + self.copied_node.license_description = "" + self.copied_node.copyright_holder = "" + self.copied_node.save() + + with patch( + "contentcuration.management.commands.fix_missing_import_sources.LicensingFixesLookup" + ) as lookup_cls: + lookup = lookup_cls.return_value + lookup.get_info.return_value = (5, "", "Khan Academy") + + call_command("fix_missing_import_sources") + + lookup.get_info.assert_not_called() + + self.copied_node.refresh_from_db() + self.assertIsNone(self.copied_node.license_id) + self.assertEqual(self.copied_node.kind_id, content_kinds.TOPIC) + self.assertEqual(self.copied_node.license_description, "") + self.assertEqual(self.copied_node.copyright_holder, "") + + +class LicensingFixesLookupTestCase(StudioTestCase): + @classmethod + def setUpTestData(cls): + cls.csv_path = ( + Path(__file__).resolve().parents[3] + / "management" + / "commands" + / "licensing_fixes_lookup.csv" ) - def test_handle__finds_for_deleted_channel(self): - self.public_channel.deleted = True - self.public_channel.save(actor_id=testdata.user().id) - call_command("fix_missing_import_sources") - - self.mock_csv_writer_instance.writerow.assert_called_once_with( - { - "channel_id": self.private_channel.id, - "channel_name": self.private_channel.name, - "contentnode_id": self.copied_node.id, - "contentnode_title": self.copied_node.title, - "public_channel_id": self.public_channel.id, - "public_channel_name": self.public_channel.name, - "public_channel_deleted": True, - } + def setUp(self): + self.lookup = LicensingFixesLookup() + + def test_load__reads_csv_and_resolves_all_licenses(self): + with self.csv_path.open("r", encoding="utf-8", newline="") as csv_file: + self.lookup.load(csv_file) + + for info in self.lookup._lookup.values(): + if info["license_name"]: + self.assertIsNotNone(info["license_id"]) + self.assertIsNotNone( + self.lookup._license_lookup.get(info["license_id"]) + ) + + def test_get_info__special_permissions(self): + with self.csv_path.open("r", encoding="utf-8", newline="") as csv_file: + self.lookup.load(csv_file) + + license_id, license_description, copyright_holder = self.lookup.get_info( + "a53592c972a8594e9b695aa127493ff6", + "exercise", + 9, + "", + "", # Special Permissions + ) + self.assertEqual(license_id, 9) + self.assertEqual( + license_description, + "Permission granted to distribute through Kolibri for non-commercial use", + ) + self.assertEqual(copyright_holder, "Khan Academy") + + def test_get_info__requires_copyright_holder(self): + with self.csv_path.open("r", encoding="utf-8", newline="") as csv_file: + self.lookup.load(csv_file) + + license_id, license_description, copyright_holder = self.lookup.get_info( + "c1f2b7e6ac9f56a2bb44fa7a48b66dce", "video", 5, "", "" # CC BY-NC-SA + ) + self.assertEqual(license_id, 5) + self.assertEqual(license_description, "") + self.assertEqual(copyright_holder, "Khan Academy") + + def test_get_info__defaults(self): + with self.csv_path.open("r", encoding="utf-8", newline="") as csv_file: + self.lookup.load(csv_file) + + license_id, license_description, copyright_holder = self.lookup.get_info( + "c51a0f842fed427c95acff9bb4a21e3c", "", None, "", "" + ) + self.assertEqual(license_id, 5) + self.assertEqual(license_description, "") + self.assertEqual(copyright_holder, "Enabling Education Network (EENET)") + + def test_get_info__broken(self): + with self.csv_path.open("r", encoding="utf-8", newline="") as csv_file: + self.lookup.load(csv_file) + + license_id, license_description, copyright_holder = self.lookup.get_info( + "237e5975bce25bf6aff398f4c17516f3", "", None, "Nothing", "Nothing" ) + self.assertIsNone(license_id) + self.assertEqual(license_description, "Nothing") + self.assertEqual(copyright_holder, "Nothing") diff --git a/contentcuration/contentcuration/tests/test_contentnodes.py b/contentcuration/contentcuration/tests/test_contentnodes.py index 329508a186..1f063264b3 100644 --- a/contentcuration/contentcuration/tests/test_contentnodes.py +++ b/contentcuration/contentcuration/tests/test_contentnodes.py @@ -824,6 +824,27 @@ def test_sync_after_no_changes(self): ) self._assert_same_files(orig_video, cloned_video) + def test_sync_but_incomplete(self): + orig_video, cloned_video = self._setup_original_and_deriative_nodes() + orig_video.license_id = None + orig_video.mark_complete() + self.assertFalse(orig_video.complete) + orig_video.save() + + self.assertTrue(cloned_video.complete) + + sync_node( + cloned_video, + sync_titles_and_descriptions=True, + sync_resource_details=True, + sync_files=True, + sync_assessment_items=True, + ) + + self.assertIsNotNone(cloned_video.license_id) + cloned_video.mark_complete() + self.assertTrue(cloned_video.complete) + def test_sync_with_subs(self): orig_video, cloned_video = self._setup_original_and_deriative_nodes() self._add_subs_to_video_node(orig_video, "fr") @@ -868,6 +889,13 @@ def _create_video_node(self, title, parent, withsubs=False): node_id="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", ) video_node = testdata.node(data, parent=parent) + video_node.license_id = 9 # Special Permissions + video_node.license_description = "Special permissions for testing" + video_node.copyright_holder = "LE" + # ensure the node is complete according to our logic + video_node.mark_complete() + self.assertTrue(video_node.complete) + video_node.save() if withsubs: self._add_subs_to_video_node(video_node, "fr") diff --git a/contentcuration/contentcuration/utils/sync.py b/contentcuration/contentcuration/utils/sync.py index 2987d1c75b..6f175d9a4f 100644 --- a/contentcuration/contentcuration/utils/sync.py +++ b/contentcuration/contentcuration/utils/sync.py @@ -53,11 +53,14 @@ def sync_node( sync_assessment_items=False, ): original_node = node.get_original_node() + if not original_node.complete: + logging.warning( + f"Refusing to sync node {node.pk} from incomplete source node: {original_node.pk}" + ) + return node if original_node.node_id != node.node_id: # Only update if node is not original logging.info( - "----- Syncing: {} from {}".format( - node.title, original_node.get_channel().name - ) + f"----- Syncing: {node.title} from {original_node.get_channel().name}" ) if sync_titles_and_descriptions: fields = [ diff --git a/docker/nginx/includes/content/_cache.conf b/docker/nginx/includes/content/_cache.conf new file mode 100644 index 0000000000..8f83f73cd8 --- /dev/null +++ b/docker/nginx/includes/content/_cache.conf @@ -0,0 +1,9 @@ +# location {} settings for /content caching +# used by files in this directory, via `include` directive + +# ignore 'expires' and 'cache-control' headers from upstream so this value is authoritative +proxy_hide_header Cache-Control; +proxy_hide_header Expires; + +# content is md5-addressed, so cache aggressively for successful responses +add_header Cache-Control "public, max-age=31536000, immutable, no-transform"; diff --git a/docker/nginx/includes/content/default.conf b/docker/nginx/includes/content/default.conf index c2c95df613..2966c3a7f6 100644 --- a/docker/nginx/includes/content/default.conf +++ b/docker/nginx/includes/content/default.conf @@ -39,7 +39,7 @@ location @nowhere { location /content/ { # check the emulator bucket first, then cloud development bucket, then fall back to production - # try_files will only use one named route, and it uses the last one. Although, we can just + # try_files will only use one named route, and it uses the last one. Although, we can't just # pass one named route, because it fails. try_files @nowhere @emulator; } diff --git a/docker/nginx/includes/content/develop-studio-content.conf b/docker/nginx/includes/content/develop-studio-content.conf index 5a1c2ed181..19f528a7be 100644 --- a/docker/nginx/includes/content/develop-studio-content.conf +++ b/docker/nginx/includes/content/develop-studio-content.conf @@ -17,16 +17,46 @@ location @production { proxy_pass https://studio-content.storage.googleapis.com; } +location @hotfixes_storage { + include /etc/nginx/includes/content/_proxy.conf; + include /etc/nginx/includes/content/_cache.conf; + + # this is the magic that allows us to intercept errors and try the next location + proxy_intercept_errors on; + recursive_error_pages on; + error_page 404 = @production_storage; + + proxy_pass https://develop-studio-content.storage.googleapis.com; +} + +location @production_storage { + include /etc/nginx/includes/content/_proxy.conf; + include /etc/nginx/includes/content/_cache.conf; + + proxy_pass https://studio-content.storage.googleapis.com; +} + location @nowhere { return 404; } +# Note on try_files +# ----------------- +# try_files will only use one named route, and it uses the last one. Although, we can't just pass +# one named route, because it fails. + +location ^~ /content/storage/ { + # ensure that the /content/ prefix is stripped from the request + rewrite ^/content/(.*)$ /$1 break; + + # check staging bucket first, then fall back to production + try_files @nowhere @hotfixes_storage; +} + location /content/ { # ensure that the /content/ prefix is stripped from the request rewrite ^/content/(.*)$ /$1 break; # check the emulator bucket first, then cloud development bucket, then fall back to production - # try_files will only use one named route, and it uses the last one. Although, we can just - # pass one named route, because it fails. try_files @nowhere @hotfixes; } diff --git a/docker/nginx/includes/content/studio-content.conf b/docker/nginx/includes/content/studio-content.conf index f59650f659..0b17852681 100644 --- a/docker/nginx/includes/content/studio-content.conf +++ b/docker/nginx/includes/content/studio-content.conf @@ -1,5 +1,16 @@ # DO NOT RENAME: this file is named after the primary bucket it proxies to +location ^~ /content/storage/ { + include /etc/nginx/includes/content/_proxy.conf; + include /etc/nginx/includes/content/_cache.conf; + + # ensure that the /content/ prefix is stripped from the request + rewrite ^/content/(.*)$ /$1 break; + + # just direct proxy to the bucket + proxy_pass https://studio-content.storage.googleapis.com; +} + location /content/ { include /etc/nginx/includes/content/_proxy.conf;