From df2314ff7a8612a0343ec686d8a13a294b5fd2eb Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Thu, 24 Sep 2026 15:26:52 -0700 Subject: [PATCH 1/3] feat: store modality on public content nodes at publish Co-Authored-By: Claude Opus 5.5 (1M context) --- .../migrations/0011_contentnode_modality.py | 39 +++++++++++++++++++ contentcuration/kolibri_public/models.py | 7 ++++ contentcuration/kolibri_public/search.py | 16 ++++++++ .../kolibri_public/tests/test_mapper.py | 21 ++++++++++ .../kolibri_public/utils/mapper.py | 7 ++-- 5 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 contentcuration/kolibri_public/migrations/0011_contentnode_modality.py diff --git a/contentcuration/kolibri_public/migrations/0011_contentnode_modality.py b/contentcuration/kolibri_public/migrations/0011_contentnode_modality.py new file mode 100644 index 0000000000..2f2a900d08 --- /dev/null +++ b/contentcuration/kolibri_public/migrations/0011_contentnode_modality.py @@ -0,0 +1,39 @@ +# Generated by Django 3.2.24 on 2026-09-24 21:28 +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ("kolibri_public", "0010_localfile_file_size_bigint"), + ] + + operations = [ + migrations.AddField( + model_name="contentnode", + name="modality", + field=models.CharField( + blank=True, + choices=[ + ("COURSE", "Course"), + ("CUSTOM_NAVIGATION", "Custom Navigation"), + ("LESSON", "Lesson"), + ("QUIZ", "Quiz"), + ("SURVEY", "Survey"), + ("UNIT", "Unit"), + ], + max_length=50, + null=True, + ), + ), + AddIndexConcurrently( + model_name="contentnode", + index=models.Index( + fields=["modality"], name="kolibri_pub_modalit_1ae095_idx" + ), + ), + ] diff --git a/contentcuration/kolibri_public/models.py b/contentcuration/kolibri_public/models.py index 154ea5cb5f..9746f4e19b 100644 --- a/contentcuration/kolibri_public/models.py +++ b/contentcuration/kolibri_public/models.py @@ -6,6 +6,7 @@ from kolibri_public.search import contentnode_bitmask_fieldnames from kolibri_public.search import contentnode_metadata_bitmasks from kolibri_public.search import has_all_labels +from le_utils.constants import modalities from mptt.managers import TreeManager from mptt.querysets import TreeQuerySet @@ -53,9 +54,15 @@ class ContentNode(base_models.ContentNode): ancestors = JSONField( default=[], null=True, blank=True, load_kwargs={"strict": False} ) + modality = models.CharField( + max_length=50, blank=True, null=True, choices=modalities.choices + ) objects = ContentNodeManager() + class Meta: + indexes = [models.Index(fields=["modality"])] + for field_name in contentnode_bitmask_fieldnames: field = models.BigIntegerField(default=0, null=True, blank=True) diff --git a/contentcuration/kolibri_public/search.py b/contentcuration/kolibri_public/search.py index ac1cef0aaa..a6639a368b 100644 --- a/contentcuration/kolibri_public/search.py +++ b/contentcuration/kolibri_public/search.py @@ -17,6 +17,7 @@ from django.db.models import Max from django.db.models import Value from django.db.models import When +from le_utils.constants import modalities from le_utils.constants.labels.accessibility_categories import ( ACCESSIBILITYCATEGORIESLIST, ) @@ -219,6 +220,21 @@ def annotate_contentnode_label_bitmasks(queryset): return annotate_label_bitmasks(queryset, contentnode_bitmask_fieldnames) +def annotate_modality(queryset): + """Update queryset to annotate `modality` field based on `options.modality`""" + queryset = queryset.filter(options__contains='"modality":') + + when_statements = [ + When( + options__contains=f'"modality": "{modality_value}"', + then=Value(modality_value), + ) + for modality_value, _ in modalities.choices + ] + + queryset.update(modality=Case(*when_statements)) + + def annotate_channelmetadata_label_bitmasks(queryset): return annotate_label_bitmasks(queryset, channelmetadata_bitmask_fieldnames) diff --git a/contentcuration/kolibri_public/tests/test_mapper.py b/contentcuration/kolibri_public/tests/test_mapper.py index 92225a2b8a..48cf1ca261 100644 --- a/contentcuration/kolibri_public/tests/test_mapper.py +++ b/contentcuration/kolibri_public/tests/test_mapper.py @@ -15,6 +15,7 @@ from kolibri_public.tests.base import OKAY_TAG from kolibri_public.utils.mapper import ChannelMapper from le_utils.constants import content_kinds +from le_utils.constants import modalities from le_utils.constants.labels.subjects import SUBJECTSLIST from contentcuration.models import Channel @@ -290,6 +291,26 @@ def test_categories_bitmask_annotation(self): ) self.assertEqual(mapper.mapped_channel.categories_bitmask_0, 1 | 4 | 16) + def test_modality_annotation(self): + with using_content_database(self.tempdb): + source = ( + kolibri_content_models.ContentNode.objects.filter( + channel_id=self.channel.id, kind=content_kinds.TOPIC + ) + .exclude(parent=None) + .first() + ) + source.options = {"modality": modalities.COURSE} + source.save() + + ChannelMapper(self.channel).run() + + mapped = kolibri_public_models.ContentNode.objects.all() + self.assertEqual(mapped.get(id=source.id).modality, modalities.COURSE) + self.assertFalse( + mapped.exclude(id=source.id).filter(modality__isnull=False).exists() + ) + def tearDown(self): # Clean up datbase connection after the test self._date_patcher.stop() diff --git a/contentcuration/kolibri_public/utils/mapper.py b/contentcuration/kolibri_public/utils/mapper.py index 0b17527b48..6929832d82 100644 --- a/contentcuration/kolibri_public/utils/mapper.py +++ b/contentcuration/kolibri_public/utils/mapper.py @@ -4,6 +4,7 @@ from kolibri_content.base_models import MAX_TAG_LENGTH from kolibri_public import models as kolibri_public_models from kolibri_public.search import annotate_contentnode_label_bitmasks +from kolibri_public.search import annotate_modality from kolibri_public.utils.annotation import set_channel_metadata_fields from le_utils.constants import content_kinds @@ -69,9 +70,9 @@ def run(self): self.mapped_channel.public = self.public self.mapped_channel.save_base(raw=True) - annotate_contentnode_label_bitmasks( - self.mapped_root.get_descendants(include_self=True) - ) + mapped_nodes = self.mapped_root.get_descendants(include_self=True) + annotate_contentnode_label_bitmasks(mapped_nodes) + annotate_modality(mapped_nodes) # Rather than set the ancestors fields after mapping, like it is done in Kolibri # here we set it during mapping as we are already recursing through the tree. From 5a78299bf933eb457c1fac2caace90c70c1e464c Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Thu, 24 Sep 2026 15:26:53 -0700 Subject: [PATCH 2/3] feat: backfill modality on existing public content nodes Co-Authored-By: Claude Opus 5.5 (1M context) --- Makefile | 2 + .../backfill_public_contentnode_modality.py | 44 +++++++++++++++++++ ...st_backfill_public_contentnode_modality.py | 42 ++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 contentcuration/kolibri_public/management/commands/backfill_public_contentnode_modality.py create mode 100644 contentcuration/kolibri_public/tests/test_backfill_public_contentnode_modality.py diff --git a/Makefile b/Makefile index 2f7ff57212..3f832c0774 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,8 @@ migrate: deploy-migrate: # studio#5974: remove at cutover. python contentcuration/manage.py backfill_column --model contentcuration.File --source-field file_size --target-field file_size_bigint + # studio#6171: remove after release. + python contentcuration/manage.py backfill_public_contentnode_modality contentnodegc: python contentcuration/manage.py garbage_collect diff --git a/contentcuration/kolibri_public/management/commands/backfill_public_contentnode_modality.py b/contentcuration/kolibri_public/management/commands/backfill_public_contentnode_modality.py new file mode 100644 index 0000000000..7722383664 --- /dev/null +++ b/contentcuration/kolibri_public/management/commands/backfill_public_contentnode_modality.py @@ -0,0 +1,44 @@ +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError +from django.db import transaction +from kolibri_public.models import ContentNode +from kolibri_public.search import annotate_modality + + +class Command(BaseCommand): + help = "Set kolibri_public ContentNode.modality from options.modality, in batches." + + def add_arguments(self, parser): + parser.add_argument("--batch-size", type=int, default=10000) + + def handle(self, *args, **options): + batch_size = options["batch_size"] + if batch_size < 1: + raise CommandError("--batch-size must be >= 1") + + # Only unfilled rows, so re-runs resume past what is already done. + unfilled = ContentNode.objects.filter( + modality__isnull=True, options__contains='"modality":' + ) + unfilled_pks = unfilled.order_by("pk").values_list("pk", flat=True) + + # Keyset paging by pk: rows whose modality value is unrecognised stay + # NULL, so the window must advance past them rather than re-query. + batch_start = unfilled_pks.first() + while batch_start is not None: + batch_end = unfilled_pks.filter(pk__gte=batch_start)[ + batch_size - 1 : batch_size + ].first() + window = unfilled.filter(pk__gte=batch_start) + if batch_end is not None: + window = window.filter(pk__lte=batch_end) + with transaction.atomic(): + annotate_modality(window) + self.stdout.write( + "backfilled through pk={}".format( + batch_start if batch_end is None else batch_end + ) + ) + if batch_end is None: + break + batch_start = unfilled_pks.filter(pk__gt=batch_end).first() diff --git a/contentcuration/kolibri_public/tests/test_backfill_public_contentnode_modality.py b/contentcuration/kolibri_public/tests/test_backfill_public_contentnode_modality.py new file mode 100644 index 0000000000..d13e30088e --- /dev/null +++ b/contentcuration/kolibri_public/tests/test_backfill_public_contentnode_modality.py @@ -0,0 +1,42 @@ +import uuid + +from django.core.management import call_command +from django.test import TestCase +from kolibri_public import models +from le_utils.constants import content_kinds +from le_utils.constants import modalities + + +class BackfillPublicContentNodeModalityTestCase(TestCase): + def _create_node(self, options): + return models.ContentNode.objects.create( + pk=uuid.uuid4().hex, + channel_id=uuid.uuid4().hex, + content_id=uuid.uuid4().hex, + kind=content_kinds.TOPIC, + title="node", + options=options, + ) + + def test_sets_modality_from_options(self): + quiz = self._create_node({"modality": modalities.QUIZ}) + course = self._create_node({"modality": modalities.COURSE}) + plain = self._create_node({}) + + call_command("backfill_public_contentnode_modality") + + self.assertEqual( + dict(models.ContentNode.objects.values_list("id", "modality")), + {quiz.id: modalities.QUIZ, course.id: modalities.COURSE, plain.id: None}, + ) + + def test_backfills_across_batches_past_unrecognised_values(self): + unknown = self._create_node({"modality": "NOT_A_MODALITY"}) + quizzes = [self._create_node({"modality": modalities.QUIZ}) for _ in range(4)] + + call_command("backfill_public_contentnode_modality", batch_size=2) + + self.assertEqual( + dict(models.ContentNode.objects.values_list("id", "modality")), + {unknown.id: None, **{q.id: modalities.QUIZ for q in quizzes}}, + ) From 43ffd379aac7535cf06a4af3c9ab9615b665d2da Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Thu, 24 Sep 2026 15:26:54 -0700 Subject: [PATCH 3/3] fix: match Kolibri's v2 contentnode and channel filter semantics Co-Authored-By: Claude Opus 5.5 (1M context) --- .../kolibri_public/tests/test_content_app.py | 148 +++++++++++++++++- contentcuration/kolibri_public/views.py | 132 +++++++++++----- 2 files changed, 230 insertions(+), 50 deletions(-) diff --git a/contentcuration/kolibri_public/tests/test_content_app.py b/contentcuration/kolibri_public/tests/test_content_app.py index cd82c03ded..2b7f104877 100644 --- a/contentcuration/kolibri_public/tests/test_content_app.py +++ b/contentcuration/kolibri_public/tests/test_content_app.py @@ -13,6 +13,7 @@ from kolibri_public.tests.base import ChannelBuilder from kolibri_public.tests.base import OKAY_TAG from le_utils.constants import content_kinds +from le_utils.constants import modalities from rest_framework.test import APITestCase from contentcuration.models import generate_storage_url @@ -165,6 +166,7 @@ def _assert_node(self, actual, expected): "lft": expected.lft, "rght": expected.rght, "tree_id": expected.tree_id, + "modality": expected.modality, "ancestors": [], "tags": list( expected.tags.all() @@ -332,6 +334,137 @@ class ContentNodeAPITestCase(ContentNodeAPIBase, APITestCase): Testcase for content API methods """ + def _list_ids(self, **params): + response = self._get(reverse("publiccontentnode-list"), data=params) + self.assertEqual(response.status_code, 200) + return {node["id"] for node in response.data} + + def _available_ids(self): + return set( + models.ContentNode.objects.filter(available=True).values_list( + "id", flat=True + ) + ) + + def _insert_second_tree(self): + # Its lft/rght ranges overlap the fixture tree's; only tree_id tells them apart. + ChannelBuilder(levels=2, num_children=2).insert_into_default_db() + models.ContentNode.objects.all().update(available=True) + + def _set_modality(self, node, modality): + models.ContentNode.objects.filter(id=node.id).update(modality=modality) + + def _mark_lesson_and_course(self): + lesson, course = ( + models.ContentNode.objects.filter(kind=content_kinds.TOPIC) + .exclude(parent=None) + .order_by("lft")[:2] + ) + self._set_modality(lesson, modalities.LESSON) + self._set_modality(course, modalities.COURSE) + return lesson, course + + def test_contentnode_modality_filter(self): + lesson, _ = self._mark_lesson_and_course() + + response = self.client.get( + reverse("publiccontentnode-list"), data={"modality": modalities.LESSON} + ) + + self.assertEqual([node["id"] for node in response.data], [lesson.id]) + self.assertEqual(response.data[0]["modality"], modalities.LESSON) + + def test_contentnode_exclude_modalities_filter(self): + lesson, course = self._mark_lesson_and_course() + + self.assertEqual( + self._list_ids(exclude_modalities=modalities.COURSE), + self._available_ids() - {course.id}, + ) + self.assertEqual( + self._list_ids( + exclude_modalities="{},{}".format(modalities.COURSE, modalities.LESSON) + ), + self._available_ids() - {course.id, lesson.id}, + ) + + def test_contentnode_exclude_course_ancestry_filter(self): + course = self.root.get_children().first() + self._set_modality(course, modalities.COURSE) + self._insert_second_tree() + descendant_ids = set(course.get_descendants().values_list("id", flat=True)) + + self.assertEqual( + self._list_ids(exclude_course_ancestry=True), + self._available_ids() - descendant_ids, + ) + self.assertEqual( + self._list_ids(exclude_course_ancestry=False), self._available_ids() + ) + + def test_contentnode_contains_quiz_filter(self): + quiz = models.ContentNode.objects.exclude(kind=content_kinds.TOPIC).first() + self._set_modality(quiz, modalities.QUIZ) + self._insert_second_tree() + + self.assertEqual( + self._list_ids(contains_quiz=True), + set(quiz.get_ancestors(include_self=True).values_list("id", flat=True)), + ) + self.assertEqual(self._list_ids(contains_quiz="false"), self._available_ids()) + + def _set_search_fixtures(self): + photo, resp = models.ContentNode.objects.exclude( + kind=content_kinds.TOPIC + ).order_by("lft")[:2] + models.ContentNode.objects.filter(id=photo.id).update( + title="Photosynthesis basics", description="Chlorophyll absorbs light" + ) + models.ContentNode.objects.filter(id=resp.id).update( + title="Energy", description="Cellular respiration" + ) + return photo.id, resp.id + + def test_contentnode_search_terms_match_across_fields(self): + photo, _ = self._set_search_fixtures() + self.assertEqual(self._list_ids(search="photosynthesis chlorophyll"), {photo}) + + def test_contentnode_search_every_term_must_match(self): + self._set_search_fixtures() + self.assertEqual(self._list_ids(search="photosynthesis respiration"), set()) + + def test_contentnode_search_quoted_phrase(self): + photo, _ = self._set_search_fixtures() + self.assertEqual(self._list_ids(search='"photosynthesis basics"'), {photo}) + self.assertEqual(self._list_ids(search='"basics photosynthesis"'), set()) + + def test_contentnode_search_drops_stopwords(self): + _, resp = self._set_search_fixtures() + self.assertEqual(self._list_ids(search="which respiration"), {resp}) + + def test_contentnode_search_keywords_param(self): + _, resp = self._set_search_fixtures() + self.assertEqual(self._list_ids(keywords="respiration"), {resp}) + + def test_contentnode_search_precedence(self): + photo, _ = self._set_search_fixtures() + self.assertEqual( + self._list_ids( + search="photosynthesis", + question="respiration", + keywords="respiration", + ), + {photo}, + ) + self.assertEqual( + self._list_ids(question="photosynthesis", keywords="respiration"), + {photo}, + ) + + def test_contentnode_search_punctuation_only_is_unfiltered(self): + self.assertEqual(self._list_ids(search="!?,"), self._available_ids()) + self.assertEqual(self._list_ids(keywords="!?,"), self._available_ids()) + def test_prerequisite_for_filter(self): response = self.client.get( reverse("publiccontentnode-list"), @@ -466,7 +599,7 @@ def test_channelmetadata_content_available_field_false(self): response = self.client.get(reverse("publicchannel-list")) self.assertEqual(response.data[0]["available"], False) - def test_channelmetadata_has_exercises_filter(self): + def test_channelmetadata_exercise_filters(self): # Has nothing else for that matter... no_exercise_channel = models.ContentNode.objects.create( pk="6a406ac66b224106aa2e93f73a94333d", @@ -492,13 +625,12 @@ def test_channelmetadata_has_exercises_filter(self): ) no_filter_response = self.client.get(reverse("publicchannel-list")) self.assertEqual(len(no_filter_response.data), 2) - with_filter_response = self.client.get( - reverse("publicchannel-list"), {"has_exercise": True} - ) - self.assertEqual(len(with_filter_response.data), 1) - self.assertEqual( - with_filter_response.data[0]["name"], self.channel_data["name"] - ) + for param in ("has_exercise", "contains_exercise"): + with self.subTest(param=param): + response = self.client.get(reverse("publicchannel-list"), {param: True}) + self.assertEqual( + [c["name"] for c in response.data], [self.channel_data["name"]] + ) def test_channelmetadata_public_filter_default_true(self): community_channel = models.ContentNode.objects.create( diff --git a/contentcuration/kolibri_public/views.py b/contentcuration/kolibri_public/views.py index 6fe7446de4..d1201006ca 100644 --- a/contentcuration/kolibri_public/views.py +++ b/contentcuration/kolibri_public/views.py @@ -9,7 +9,6 @@ import logging import re from collections import OrderedDict -from functools import reduce from uuid import UUID from django.core.exceptions import ValidationError @@ -22,6 +21,8 @@ from django.utils.cache import patch_cache_control from django.utils.cache import patch_response_headers from django.utils.decorators import method_decorator +from django.utils.text import smart_split +from django.utils.text import unescape_string_literal from django.utils.translation import ugettext as _ from django.views.decorators.http import last_modified from django_filters.rest_framework import BaseInFilter @@ -38,11 +39,13 @@ from kolibri_public.stopwords import stopwords_set from le_utils.constants import content_kinds from le_utils.constants import library as library_constants +from le_utils.constants import modalities from rest_framework import status from rest_framework.decorators import action from rest_framework.filters import SearchFilter from rest_framework.permissions import AllowAny from rest_framework.response import Response +from rest_framework.serializers import CharField from contentcuration.middleware.locale import locale_exempt from contentcuration.middleware.session import session_exempt @@ -86,9 +89,6 @@ def wrapper_func(*args, **kwargs): return locale_exempt(session_exempt(wrapper_func)) -MODALITIES = set(["QUIZ"]) - - def bitmask_contains_and(queryset, name, value): """ A filtering method that filters instances matching all provided @@ -105,6 +105,10 @@ class CharInFilter(BaseInFilter, CharFilter): pass +class ChoiceInFilter(BaseInFilter, ChoiceFilter): + pass + + class ChannelMetadataFilter(FilterSet): def __init__(self, data=None, *args, **kwargs): # if filterset is bound, use initial values as defaults @@ -119,6 +123,9 @@ def __init__(self, data=None, *args, **kwargs): available = BooleanFilter(method="filter_available", label="Available") has_exercise = BooleanFilter(method="filter_has_exercise", label="Has exercises") + contains_exercise = BooleanFilter( + method="filter_has_exercise", label="Has exercises" + ) categories = CharFilter(method=bitmask_contains_and, label="Categories") countries = CharInFilter(field_name="countries", label="Countries") public = BooleanFilter(field_name="public", label="Public", initial=True) @@ -129,6 +136,7 @@ class Meta: fields = ( "available", "has_exercise", + "contains_exercise", "categories", "countries", "public", @@ -294,28 +302,16 @@ def labels(self, request): "accessibility_labels", "categories", "learner_needs", - "keywords", "channels", "languages", "tree_id", "lft__gt", "rght__lt", + "modality", + "exclude_course_ancestry", ] -# return the result of and-ing a list of queries -def intersection(queries): - if queries: - return reduce(lambda x, y: x & y, queries) - return None - - -def union(queries): - if queries: - return reduce(lambda x, y: x | y, queries) - return None - - class ContentNodeFilter(FilterSet): ids = UUIDInFilter(field_name="id") kind = ChoiceFilter( @@ -327,14 +323,13 @@ class ContentNodeFilter(FilterSet): parent = UUIDFilter("parent") parent__isnull = BooleanFilter(field_name="parent", lookup_expr="isnull") include_coach_content = BooleanFilter(method="filter_include_coach_content") - contains_quiz = CharFilter(method="filter_contains_quiz") + contains_quiz = BooleanFilter(method="filter_contains_quiz") grade_levels = CharFilter(method=bitmask_contains_and) resource_types = CharFilter(method=bitmask_contains_and) learning_activities = CharFilter(method=bitmask_contains_and) accessibility_labels = CharFilter(method=bitmask_contains_and) categories = CharFilter(method=bitmask_contains_and) learner_needs = CharFilter(method=bitmask_contains_and) - keywords = CharFilter(method="filter_keywords") channels = UUIDInFilter(field_name="channel_id") languages = CharInFilter(field_name="lang_id") categories__isnull = BooleanFilter(field_name="categories", lookup_expr="isnull") @@ -343,6 +338,10 @@ class ContentNodeFilter(FilterSet): authors = CharFilter(method="filter_by_authors") tags = CharFilter(method="filter_by_tags") descendant_of = UUIDFilter(method="filter_descendant_of") + exclude_modalities = ChoiceInFilter( + field_name="modality", choices=modalities.choices, exclude=True + ) + exclude_course_ancestry = BooleanFilter(method="filter_exclude_course_ancestry") class Meta: model = models.ContentNode @@ -419,30 +418,78 @@ def filter_include_coach_content(self, queryset, name, value): return queryset return queryset.filter(coach_content=False) + def filter_exclude_course_ancestry(self, queryset, name, value): + if not value: + return queryset + has_course_ancestor = models.ContentNode.objects.filter( + modality=modalities.COURSE, + available=True, + tree_id=OuterRef("tree_id"), + lft__lt=OuterRef("lft"), + rght__gt=OuterRef("rght"), + ) + return queryset.exclude(Exists(has_course_ancestor)) + def filter_contains_quiz(self, queryset, name, value): - if value: - quizzes = models.ContentNode.objects.filter( - options__contains='"modality": "QUIZ"' - ).get_ancestors(include_self=True) - return queryset.filter(pk__in=quizzes.values_list("pk", flat=True)) - return queryset - - def filter_keywords(self, queryset, name, value): - # all words with punctuation removed - all_words = [w for w in re.split('[?.,!";: ]', value) if w] - # words in all_words that are not stopwords - critical_words = [w for w in all_words if w not in stopwords_set] - words = critical_words if critical_words else all_words - query = union( - [ - # all critical words in title - intersection([Q(title__icontains=w) for w in words]), - # all critical words in description - intersection([Q(description__icontains=w) for w in words]), - ] + if not value: + return queryset + # Not get_ancestors() over quizzes: unusably slow (kolibri#13267). + quiz_descendants = models.ContentNode.objects.filter( + modality=modalities.QUIZ, + available=True, + tree_id=OuterRef("tree_id"), + lft__gte=OuterRef("lft"), + rght__lte=OuterRef("rght"), + ) + return queryset.filter(Exists(quiz_descendants)) + + +def search_smart_split(search_terms): + """ + Returns sanitized search terms as a list. + Vendored and modified from https://github.com/encode/django-rest-framework/blob/main/rest_framework/filters.py#L23 + to add splitting by more punctuation types. + """ + split_terms = [] + for term in smart_split(search_terms): + # trim commas to avoid bad matching for quoted phrases + term = term.strip(",") + if term.startswith(('"', "'")) and term[0] == term[-1]: + # quoted phrases are kept together without any other split + split_terms.append(unescape_string_literal(term)) + else: + # non-quoted tokens are split by ?.,!;:, keeping only non-empty ones + for sub_term in re.split("[?.,!;:]", term): + if sub_term: + split_terms.append(sub_term.strip()) + return split_terms + + +class ContentNodeSearchFilter(SearchFilter): + def get_search_fields(self, view, request): + return ["title", "description"] + + def get_cleaned_search_value(self, request): + value = request.query_params.get( + self.search_param, + request.query_params.get( + "question", request.query_params.get("keywords", "") + ), ) + field = CharField(trim_whitespace=False, allow_blank=True) + return field.run_validation(value) - return queryset.filter(query) + def get_search_terms(self, request): + """ + Search terms are set by a ?search=... query parameter, + and may be whitespace delimited. + For backwards compatibility, we also allow the question and keywords + parameters, but search will take precedence. + """ + cleaned_value = self.get_cleaned_search_value(request) + split_terms = search_smart_split(cleaned_value) + critical_terms = [w for w in split_terms if w not in stopwords_set] + return critical_terms if critical_terms else split_terms def map_file(file): @@ -467,7 +514,7 @@ class BaseContentNodeMixin(object): serialization for ContentNodes. """ - filter_backends = (DjangoFilterBackend,) + filter_backends = (DjangoFilterBackend, ContentNodeSearchFilter) # Update from filter_class to filterset_class for newer version of Django Filters filterset_class = ContentNodeFilter # Add an explicit allow any permission class to override the Studio default @@ -502,6 +549,7 @@ class BaseContentNodeMixin(object): "categories", "duration", "ancestors", + "modality", ) field_map = {