diff options
author | Matthew Lemon <y@yulqen.org> | 2024-05-14 12:53:28 +0100 |
---|---|---|
committer | Matthew Lemon <y@yulqen.org> | 2024-05-14 12:53:28 +0100 |
commit | 46f11648d902b22a177b878e35d6049a7a127ce7 (patch) | |
tree | f59f6630717bc9097c988a6d8d3eebe4ad548f1d | |
parent | b5e2c4b9a7aab20db6dd6072a01abd114e8e55de (diff) |
Can now upload to Spaces
-rw-r--r-- | .envs/.local/.django | 14 | ||||
-rw-r--r-- | .envs/.local/.postgres | 7 | ||||
-rw-r--r-- | .gitignore | 2 | ||||
-rw-r--r-- | compose/local/django/Dockerfile | 2 | ||||
-rw-r--r-- | compose/production/django/Dockerfile | 1 | ||||
-rw-r--r-- | config/settings/base.py | 32 | ||||
-rw-r--r-- | config/urls.py | 2 | ||||
-rw-r--r-- | pyblackbird_cc/resources/admin.py | 14 | ||||
-rw-r--r-- | pyblackbird_cc/resources/forms.py | 106 | ||||
-rw-r--r-- | pyblackbird_cc/resources/migrations/0001_initial.py | 90 | ||||
-rw-r--r-- | pyblackbird_cc/resources/models.py | 104 | ||||
-rw-r--r-- | pyblackbird_cc/resources/services.py | 57 | ||||
-rw-r--r-- | pyblackbird_cc/resources/urls.py | 16 | ||||
-rw-r--r-- | pyblackbird_cc/resources/views.py | 358 | ||||
-rw-r--r-- | pyblackbird_cc/static/css/project.css | 40 | ||||
-rw-r--r-- | pyblackbird_cc/templates/base.html | 70 | ||||
-rw-r--r-- | pyblackbird_cc/templates/resources/hx_download_button.html | 3 | ||||
-rw-r--r-- | pyblackbird_cc/templates/resources/resource_create.html | 21 | ||||
-rw-r--r-- | pyblackbird_cc/templates/resources/resource_detail.html | 50 | ||||
-rw-r--r-- | pyblackbird_cc/templates/resources/resource_list.html | 61 | ||||
-rw-r--r-- | requirements/base.txt | 12 |
21 files changed, 1000 insertions, 62 deletions
diff --git a/.envs/.local/.django b/.envs/.local/.django deleted file mode 100644 index e20db73..0000000 --- a/.envs/.local/.django +++ /dev/null @@ -1,14 +0,0 @@ -# General -# ------------------------------------------------------------------------------ -USE_DOCKER=yes -IPYTHONDIR=/app/.ipython -# Redis -# ------------------------------------------------------------------------------ -REDIS_URL=redis://redis:6379/0 - -# Celery -# ------------------------------------------------------------------------------ - -# Flower -CELERY_FLOWER_USER=GtKEXvuJJrEWITDgEQboQqwlxYBwjUfa -CELERY_FLOWER_PASSWORD=Scv4VgYqnUjkkaS2bP4rk6Txn3eFqTYuuXRQRMLCO14BSehv7tzaGgYlPLxRcsyE diff --git a/.envs/.local/.postgres b/.envs/.local/.postgres deleted file mode 100644 index 302197d..0000000 --- a/.envs/.local/.postgres +++ /dev/null @@ -1,7 +0,0 @@ -# PostgreSQL -# ------------------------------------------------------------------------------ -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 -POSTGRES_DB=pyblackbird_cc -POSTGRES_USER=TyGbwdJQNsWKWloWSdhHyTmXiKrvZeij -POSTGRES_PASSWORD=p9yz0UNLt1YzhvfZFl24vwOeTvmmfyfrUyFuEjqRa2QlHQaR0OixvDORr04MIrjz @@ -332,4 +332,4 @@ pyblackbird_cc/media/ .ipython/ .env .envs/* -!.envs/.local/ +.envs/.local/ diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 5f5f4e5..553971b 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -47,6 +47,7 @@ RUN groupadd --gid 1000 dev-user \ # Install required system dependencies RUN apt-get update && apt-get install --no-install-recommends -y \ + libmagic1 \ # psycopg dependencies libpq-dev \ # Translations dependencies @@ -71,7 +72,6 @@ COPY ./compose/local/django/start /start RUN sed -i 's/\r$//g' /start RUN chmod +x /start - COPY ./compose/local/django/celery/worker/start /start-celeryworker RUN sed -i 's/\r$//g' /start-celeryworker RUN chmod +x /start-celeryworker diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index 502c16e..6e816f1 100644 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -40,6 +40,7 @@ RUN addgroup --system django \ # Install required system dependencies RUN apt-get update && apt-get install --no-install-recommends -y \ + libmagic1 \ # psycopg dependencies libpq-dev \ # Translations dependencies diff --git a/config/settings/base.py b/config/settings/base.py index 3c11aee..16d79eb 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -314,6 +314,38 @@ SOCIALACCOUNT_ADAPTER = "pyblackbird_cc.users.adapters.SocialAccountAdapter" # https://docs.allauth.org/en/latest/socialaccount/configuration.html SOCIALACCOUNT_FORMS = {"signup": "pyblackbird_cc.users.forms.UserSocialSignupForm"} +# STORAGES +# ------------------------------------------------------------------------------ +# https://django-storages.readthedocs.io/en/latest/#installation +INSTALLED_APPS += ["storages"] +# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings +# AWS_ACCESS_KEY_ID = env("DJANGO_AWS_ACCESS_KEY_ID") +AWS_ACCESS_KEY_ID = env("SPACES_KEY") +# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings +# AWS_SECRET_ACCESS_KEY = env("DJANGO_AWS_SECRET_ACCESS_KEY") +AWS_SECRET_ACCESS_KEY = env("SPACES_SECRET") +# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings +# AWS_STORAGE_BUCKET_NAME = env("DJANGO_AWS_STORAGE_BUCKET_NAME") +AWS_STORAGE_BUCKET_NAME = env("SPACES_BUCKET_NAME") +# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings +AWS_QUERYSTRING_AUTH = False +# DO NOT change these unless you know what you're doing. +_AWS_EXPIRY = 60 * 60 * 24 * 7 +# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings +AWS_S3_OBJECT_PARAMETERS = { + "CacheControl": f"max-age={_AWS_EXPIRY}, s-maxage={_AWS_EXPIRY}, must-revalidate", +} +# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings +AWS_S3_MAX_MEMORY_SIZE = env.int( + "DJANGO_AWS_S3_MAX_MEMORY_SIZE", + default=100_000_000, # 100MB +) +# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings +AWS_S3_REGION_NAME = env("DJANGO_AWS_S3_REGION_NAME", default=None) +# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#cloudfront +AWS_S3_CUSTOM_DOMAIN = env("DJANGO_AWS_S3_CUSTOM_DOMAIN", default=None) +aws_s3_domain = AWS_S3_CUSTOM_DOMAIN or f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" +AWS_S3_ENDPOINT_URL = env("SPACES_ENDPOINT_URL") # Your stuff... # ------------------------------------------------------------------------------ diff --git a/config/urls.py b/config/urls.py index bc672a5..270a8ad 100644 --- a/config/urls.py +++ b/config/urls.py @@ -20,7 +20,7 @@ urlpatterns = [ path("users/", include("pyblackbird_cc.users.urls", namespace="users")), path("accounts/", include("allauth.urls")), # Your stuff: custom urls includes go here - # ... + path("resources/", include("pyblackbird_cc.resources.urls", namespace="resources")), # Media files *static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT), ] diff --git a/pyblackbird_cc/resources/admin.py b/pyblackbird_cc/resources/admin.py index 846f6b4..1140c35 100644 --- a/pyblackbird_cc/resources/admin.py +++ b/pyblackbird_cc/resources/admin.py @@ -1 +1,15 @@ # Register your models here. +from django.contrib import admin + +from pyblackbird_cc.resources.models import ResourceCategory +from pyblackbird_cc.resources.models import ResourceType + + +@admin.register(ResourceType) +class ResourceTypeAdmin(admin.ModelAdmin): + pass + + +@admin.register(ResourceCategory) +class ResourceCategoryAdmin(admin.ModelAdmin): + pass diff --git a/pyblackbird_cc/resources/forms.py b/pyblackbird_cc/resources/forms.py new file mode 100644 index 0000000..c3b938d --- /dev/null +++ b/pyblackbird_cc/resources/forms.py @@ -0,0 +1,106 @@ +import logging + +import magic +from django import forms + +from .models import ResourceCategory +from .models import ResourceType + +logger = logging.getLogger(__name__) + + +class ResourceCreateForm(forms.Form): + error_css_class = "error" + required_css_class = "required" + + name = forms.CharField( + max_length=255, + help_text="Concisely describe what the resource is, aiming for 35-45 characters. eg: ‘Fractions KS2 Worksheet and Answers.'", + ) + description = forms.CharField( + max_length=1000, + widget=forms.Textarea, + help_text="This is your opportunity to clearly explain what your resource is all about! It’s worth remembering that you are using the space to communicate to two different audiences. Firstly, think about what fellow teachers would like to know, such as exactly what the resource contains and how it could be used in the classroom. Secondly, the words you include on this page are also talking to internal and external search engines. External search engines, like Google, show the first 155 characters of the resource description, so make sure you take advantage of these characters by using lots of relevant keywords as part of an enticing pitch.", + ) + resource_type = forms.ModelChoiceField(queryset=ResourceType.objects.all()) + age_range = forms.ChoiceField( + choices=[ + ("3-5", "3-5"), + ("5-7", "5-7"), + ("7-11", "7-11"), + ("11-14", "11-14"), + ("14-16", "14-16"), + ("16+", "16+"), + ("Age not applicable", "Age not applicable"), + ], + help_text="Try to be accurate in your choice of age range so that your resource shows up in the correct searches. (Although we don't have searches yet!)", + ) + curriculum = forms.ChoiceField( + choices=[ + ("No curriculum", "No curriculum"), + ("English", "English"), + ("Scottish", "Scottish"), + ], + ) + main_resource_category = forms.ModelChoiceField( + queryset=ResourceCategory.objects.all(), + help_text="Categorise your resource by subject so it shows up in the correct searches. It's a good idea to limit the number of subjects you select to one or two to make your resource easier to find.", + ) + additional_resource_category = forms.ModelChoiceField( + queryset=ResourceCategory.objects.all(), + required=False, + ) + pdf_files = forms.FileField( + widget=forms.TextInput( + attrs={ + "multiple": True, + "type": "File", + "required": True, + }, + ), + required=False, + help_text="You can multi-select up to 20 .pdf files here.", + label="PDF files", + ) + thumbnail_files = forms.FileField( + widget=forms.TextInput( + attrs={ + "multiple": True, + "type": "File", + "required": True, + }, + ), + required=False, + label="Cover images", + help_text="Your cover image will be displayed in the search results and as the first image on your resource page in the preview function. It is important to add an eye catching cover image that gives other teachers an idea about what your resource contains. You can multi-select up to 5 .png or .jpg files here.", + ) + pdf_files.widget.attrs.update({"class": "file_upload", "accept": ".pdf"}) + thumbnail_files.widget.attrs.update({"class": "file_upload", "accept": ".png,.jpg"}) + + def clean_thumbnail_files(self): + thumbnail_files = self.files.getlist("thumbnail_files") + if not thumbnail_files: + raise forms.ValidationError("Please select at least one thumbnail file.") + acceptable = ["image/png", "image/jpeg"] + for f in thumbnail_files: + content_type = magic.from_buffer(f.file.read(), mime=True) + f.file.seek(0) + if content_type not in acceptable: + raise forms.ValidationError("Please select only PNG or JPG files.") + if len(thumbnail_files) > 5: + raise forms.ValidationError("Please select up to 5 files.") + return thumbnail_files + + def clean_pdf_files(self): + pdf_files = self.files.getlist("pdf_files") + if not pdf_files: + raise forms.ValidationError("Please select at least one PDF file.") + acceptable = ["application/pdf"] + for f in pdf_files: + content_type = magic.from_buffer(f.file.read(), mime=True) + f.file.seek(0) + if content_type not in acceptable: + raise forms.ValidationError("Please select only PDF files.") + if len(pdf_files) > 20: + raise forms.ValidationError("Please select up to 20 PDF files.") + return pdf_files diff --git a/pyblackbird_cc/resources/migrations/0001_initial.py b/pyblackbird_cc/resources/migrations/0001_initial.py new file mode 100644 index 0000000..812c98f --- /dev/null +++ b/pyblackbird_cc/resources/migrations/0001_initial.py @@ -0,0 +1,90 @@ +# Generated by Django 5.0.4 on 2024-05-13 21:41 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='PDFResource', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('file_name', models.CharField(max_length=255)), + ('file_size', models.IntegerField()), + ], + ), + migrations.CreateModel( + name='Resource', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('thumbnail_filenames', models.JSONField(default=list, verbose_name='Thumbnail filenames')), + ('description', models.TextField(max_length=1000)), + ('age_range', models.CharField(choices=[('3-5', '3-5'), ('5-7', '5-7'), ('7-11', '7-11'), ('11-14', '11-14'), ('14-16', '14-16'), ('16+', '16+'), ('Age not applicable', 'Age not applicable')], default='5-7', max_length=20)), + ('curriculum', models.CharField(blank=True, choices=[('No curriculum', 'No curriculum'), ('English', 'English'), ('Scottish', 'Scottish')], default='English', max_length=20, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='ResourceCategory', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + options={ + 'verbose_name_plural': 'Resource Categories', + }, + ), + migrations.CreateModel( + name='ResourceType', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='PDFPageSnapshot', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('file_name', models.CharField(max_length=255)), + ('pdf_file', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pdf_page_snapshots', to='resources.pdfresource')), + ], + ), + migrations.AddField( + model_name='pdfresource', + name='resource', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pdf_resources', to='resources.resource'), + ), + migrations.AddField( + model_name='resource', + name='additional_resource_category', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='additional_resource_category', to='resources.resourcecategory'), + ), + migrations.AddField( + model_name='resource', + name='main_resource_category', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='main_resource_category', to='resources.resourcecategory'), + ), + migrations.AddField( + model_name='resource', + name='resource_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='resources.resourcetype'), + ), + migrations.AlterUniqueTogether( + name='pdfresource', + unique_together={('resource', 'file_name')}, + ), + ] diff --git a/pyblackbird_cc/resources/models.py b/pyblackbird_cc/resources/models.py index 6b20219..68f0637 100644 --- a/pyblackbird_cc/resources/models.py +++ b/pyblackbird_cc/resources/models.py @@ -1 +1,105 @@ +from django.db import models + + # Create your models here. +class Resource(models.Model): + name = models.CharField(max_length=255, null=False) + thumbnail_filenames = models.JSONField( + null=False, + verbose_name="Thumbnail filenames", + default=list, + ) + resource_type = models.ForeignKey("ResourceType", on_delete=models.CASCADE) + main_resource_category = models.ForeignKey( + "ResourceCategory", + on_delete=models.CASCADE, + null=False, + related_name="main_resource_category", + ) + additional_resource_category = models.ForeignKey( + "ResourceCategory", + on_delete=models.CASCADE, + null=True, + related_name="additional_resource_category", + ) + description = models.TextField(max_length=1000, null=False, blank=False) + age_range = models.CharField( + max_length=20, + null=False, + default="5-7", + blank=False, + choices=[ + ("3-5", "3-5"), + ("5-7", "5-7"), + ("7-11", "7-11"), + ("11-14", "11-14"), + ("14-16", "14-16"), + ("16+", "16+"), + ("Age not applicable", "Age not applicable"), + ], + ) + curriculum = models.CharField( + max_length=20, + null=True, + default="English", + blank=True, + choices=[ + ("No curriculum", "No curriculum"), + ("English", "English"), + ("Scottish", "Scottish"), + ], + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.name + + +class ResourceType(models.Model): + name = models.CharField(max_length=255, null=False) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.name + + +class ResourceCategory(models.Model): + name = models.CharField(max_length=255, null=False) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + verbose_name_plural = "Resource Categories" + + def __str__(self): + return self.name + + +class PDFResource(models.Model): + resource = models.ForeignKey( + "Resource", + on_delete=models.CASCADE, + null=False, + related_name="pdf_resources", + ) + file_name = models.CharField(max_length=255, null=False) + file_size = models.IntegerField(null=False) + + class Meta: + unique_together = ("resource", "file_name") + + def snapshot_file_names(self): + return [f.file_name for f in self.pdf_page_snapshots.all()] + + +class PDFPageSnapshot(models.Model): + name = models.CharField(max_length=255, null=False) + file_name = models.CharField(max_length=255, null=False) + pdf_file = models.ForeignKey( + "PDFResource", + on_delete=models.CASCADE, + null=False, + related_name="pdf_page_snapshots", + ) diff --git a/pyblackbird_cc/resources/services.py b/pyblackbird_cc/resources/services.py new file mode 100644 index 0000000..b33afcf --- /dev/null +++ b/pyblackbird_cc/resources/services.py @@ -0,0 +1,57 @@ +import dataclasses +import os.path +import tempfile + +import pypdfium2 as pdfium + + +@dataclasses.dataclass +class PDFMetadata: + file_name: str + file_size: int + n_pages: int + + +def get_pdf_metadata_from_path(file_path: str) -> PDFMetadata: + """ + This function returns the metadata of a PDF file + :param file_path: + :return: PDFMetadata + """ + if file_path is None: + raise ValueError("file_path cannot be None") + if not os.path.isfile(file_path): + raise ValueError(f"file_path must be a file. {file_path} is not a file.") + pdf = pdfium.PdfDocument(file_path) + n_pages = len(pdf) + file_size = os.path.getsize(file_path) + pdf.close() + return PDFMetadata(file_name=file_path, file_size=file_size, n_pages=n_pages) + + +def export_pages_as_images(file_path: str) -> list[str]: + """ + This function exports the pages of a PDF file as JPEG images. + :param file_path: + :return: List of paths to the JPEG images + """ + if file_path is None: + raise ValueError("file_path cannot be None") + output_dir = tempfile.mkdtemp() # Create a temporary directory + try: + pdf = pdfium.PdfDocument(file_path) + # get the file_name of this PDF file at file_path + file_name = os.path.basename(file_path) + image_paths = [] + for i in range(len(pdf)): + page = pdf[i] + image = page.render(scale=0.6).to_pil() + image_path = os.path.join(output_dir, f"{file_name}_{i:03d}.jpg") + image.save(image_path) + image_paths.append(image_path) + pdf.close() + return image_paths + finally: + # Optionally handle cleanup later or elsewhere in your code + # Remove later with shutil.rmtree(output_dir) + pass diff --git a/pyblackbird_cc/resources/urls.py b/pyblackbird_cc/resources/urls.py new file mode 100644 index 0000000..30c5f99 --- /dev/null +++ b/pyblackbird_cc/resources/urls.py @@ -0,0 +1,16 @@ +from django.urls import path + +from . import views + +app_name = "resources" +urlpatterns = [ + path("", views.index, name="index"), + path("create/", views.create_resource, name="create_resource"), + path("resource/<int:resource_id>", views.resource_detail, name="resource_detail"), +] + +htmx_patterns = [ + path("hx-download-btn/", views.hx_download_button, name="hx_download_button"), +] + +urlpatterns += htmx_patterns diff --git a/pyblackbird_cc/resources/views.py b/pyblackbird_cc/resources/views.py index 60f00ef..b09f37e 100644 --- a/pyblackbird_cc/resources/views.py +++ b/pyblackbird_cc/resources/views.py @@ -1 +1,357 @@ -# Create your views here. +import logging +import os +import tempfile +from collections.abc import Generator +from dataclasses import dataclass + +import boto3 +from botocore.exceptions import ClientError +from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.db import transaction +from django.shortcuts import get_object_or_404 +from django.shortcuts import redirect +from django.shortcuts import render + +from . import services +from .forms import ResourceCreateForm +from .models import PDFPageSnapshot +from .models import PDFResource +from .models import Resource + +logger = logging.getLogger(__name__) + + +# I want to create a dataclass here to hold the resource information to pass to the view +@dataclass +class ResourceInfo: + id: int + name: str + main_resource_category_name: str + additional_resource_category_name: str | None + pdf_filenames: list[str] + pdf_urls: list[str] + snapshot_urls: dict[str, list[str]] + thumbnail_filenames: list[str] + thumbnail_urls: list[str] + created: str + updated: str + + +def _extract_metadata_from_resource(resource_obj) -> ResourceInfo | None: + """ + This function extracts the resource information from the model object and returns it as a ResourceInfo object + :param resource_obj: + :return: + """ + pdf_resource_filenames = [ + x.file_name for x in PDFResource.objects.filter(resource=resource_obj).all() + ] + pdf_resources = PDFResource.objects.filter(resource=resource_obj).all() + snapshot_dict = {} + for p in pdf_resources: + snapshot_dict[p.file_name] = [ + x.file_name for x in PDFPageSnapshot.objects.filter(pdf_file=p).all() + ] + snapshot_url_dict = {} + # Iterate through the snapshot dict and generate the URLs + for k, v in snapshot_dict.items(): + snapshot_url_dict[k] = [ + get_presigned_obj_url( + settings.AWS_STORAGE_BUCKET_NAME, + f"snapshotted_pages/{f}", + ) + for f in v + ] + pdf_urls = [ + get_presigned_obj_url(settings.AWS_STORAGE_BUCKET_NAME, f"pdfuploads/{f}") + for f in pdf_resource_filenames + ] + thumbnail_urls = [ + get_presigned_obj_url(settings.AWS_STORAGE_BUCKET_NAME, f"thumbnails/{f}") + for f in resource_obj.thumbnail_filenames + ] + try: + if resource_obj.additional_resource_category: + arc_name = resource_obj.additional_resource_category.name + else: + arc_name = None + return ResourceInfo( + id=resource_obj.id, + name=resource_obj.name, + main_resource_category_name=resource_obj.main_resource_category.name, + additional_resource_category_name=arc_name, + pdf_filenames=pdf_resource_filenames, + pdf_urls=pdf_urls, + snapshot_urls=snapshot_url_dict, + thumbnail_filenames=resource_obj.thumbnail_filenames, + thumbnail_urls=thumbnail_urls, + created=resource_obj.created_at.strftime("%Y-%m-%d %H:%M:%S"), + updated=resource_obj.updated_at.strftime("%Y-%m-%d %H:%M:%S"), + ) + except Exception as e: + logging.exception(f"Error extracting resource information: {e}") + return None + + +@login_required +def index(request): + resource_objs = Resource.objects.all() + resource_list = [_extract_metadata_from_resource(r) for r in resource_objs] + context = {"resource_list": resource_list} + return render(request, "resources/resource_list.html", context) + + +def get_presigned_obj_url(bucket_name, obj_name, expiration=3600) -> str | None: + client = boto3.client( + "s3", + endpoint_url=settings.AWS_S3_ENDPOINT_URL, + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + region_name=settings.AWS_S3_REGION_NAME, + ) + logger.info("Client created", extra={"client": client}) + try: + response = client.generate_presigned_url( + "get_object", + Params={"Bucket": bucket_name, "Key": obj_name}, + ExpiresIn=expiration, + ) + except ClientError as e: + logger.error(e) + return None + return response + + +def upload_to_s3(pdf_files, thumbnail_files, snappedshotted_pages) -> bool: + session = boto3.Session() + client = session.client( + "s3", + endpoint_url=settings.AWS_S3_ENDPOINT_URL, + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + region_name=settings.AWS_S3_REGION_NAME, + ) + + try: + for pdf_file in pdf_files: + logger.info(f"Uploading {pdf_file.name} to S3") + client.upload_fileobj( + pdf_file, + settings.AWS_STORAGE_BUCKET_NAME, + f"pdfuploads/{pdf_file.name}", + ) + for f in thumbnail_files: + logger.info(f"Uploading {f.name} to S3") + client.upload_fileobj( + f, + settings.AWS_STORAGE_BUCKET_NAME, + f"thumbnails/{f.name}", + ) + if len(snappedshotted_pages[0]) == 1: + for img in snappedshotted_pages[0]: + logger.info(f"Uploading {img} to S3") + client.upload_file( + img, + settings.AWS_STORAGE_BUCKET_NAME, + f"snapshotted_pages/{os.path.basename(img)}", + ) + else: + for lst in snappedshotted_pages: + for img in lst: + logger.info(f"Uploading {img} to S3") + client.upload_file( + img, + settings.AWS_STORAGE_BUCKET_NAME, + f"snapshotted_pages/{os.path.basename(img)}", + ) + return True + except ClientError as e: + logging.exception(f"Error uploading files to S3: {e}") + return False + + +def _write_pdf_to_tempdir(f) -> str: + temp_dir = tempfile.mkdtemp() + file_path = os.path.join(temp_dir, f.name) + + with open(file_path, "wb") as destination: + for chunk in f.chunks(): + destination.write(chunk) + return file_path + + +def create_metadata( + pdf_files, +) -> Generator[tuple[services.PDFMetadata, str], None, None]: + with tempfile.TemporaryDirectory() as temp_dir: + for pdf_file in pdf_files: + file_path = os.path.join(temp_dir, pdf_file.name) + + with open(file_path, "wb") as temp_file: + for chunk in pdf_file.chunks(): + temp_file.write(chunk) + + metadata = services.get_pdf_metadata_from_path(file_path) + snapshot_images = services.export_pages_as_images(file_path) + + yield metadata, snapshot_images + + +@transaction.atomic +def create_resource_objects(resource, metadata_generator, thumbnail_files): + for metadata, snapshot_images in metadata_generator: + pdf_resource = PDFResource.objects.create( + resource=resource, + file_name=os.path.basename(metadata.file_name), + file_size=metadata.file_size, + ) + + for snapshot_image in snapshot_images: + PDFPageSnapshot.objects.create( + name="test", + file_name=os.path.basename(snapshot_image), + pdf_file=pdf_resource, + ) + + resource.thumbnail_filenames = [f.name for f in thumbnail_files] + resource.save() + + +@login_required +def create_resource(request): + if request.method == "POST": + form = ResourceCreateForm(request.POST, request.FILES) + + if form.is_valid(): + pdf_files = form.cleaned_data["pdf_files"] + thumbnail_files = form.cleaned_data["thumbnail_files"] + name = form.cleaned_data["name"] + description = form.cleaned_data["description"] + resource_type = form.cleaned_data["resource_type"] + age_range = form.cleaned_data["age_range"] + curriculum = form.cleaned_data["curriculum"] + main_resource_category = form.cleaned_data["main_resource_category"] + additional_resource_category = form.cleaned_data[ + "additional_resource_category" + ] + + try: + resource = Resource.objects.create( + name=name, + description=description, + resource_type=resource_type, + age_range=age_range, + curriculum=curriculum, + main_resource_category=main_resource_category, + additional_resource_category=additional_resource_category, + ) + + metadata_generator = create_metadata(pdf_files) + snapshotted_pages = [] + + for metadata, snapshot_images in metadata_generator: + pdf_resource = PDFResource.objects.create( + resource=resource, + file_name=os.path.basename(metadata.file_name), + file_size=metadata.file_size, + ) + + for snapshot_image in snapshot_images: + PDFPageSnapshot.objects.create( + name="test", + file_name=os.path.basename(snapshot_image), + pdf_file=pdf_resource, + ) + + snapshotted_pages.append(snapshot_images) + + resource.thumbnail_filenames = [f.name for f in thumbnail_files] + resource.save() + + # Reset the file pointers for pdf_files + for pdf_file in pdf_files: + pdf_file.seek(0) + + if not upload_to_s3(pdf_files, thumbnail_files, snapshotted_pages): + raise Exception("Error uploading files to S3") + + return redirect("resources:resource_detail", resource_id=resource.id) + except Exception as e: + logger.error(f"Error creating resource: {e}") + form.add_error(None, "An error occurred while creating the resource.") + else: + # extract form errors + errors = {} + for field in form: + if field.errors: + errors[field.name] = field.errors + + # add non-field errors + if form.non_field_errors(): + errors["non_field_errors"] = form.non_field_errors() + + # render form with errors + return render( + request, + "resources/resource_create.html", + {"form": form, "errors": errors}, + ) + else: + form = ResourceCreateForm() + + return render(request, "resources/resource_create.html", {"form": form}) + + +@login_required +def resource_detail(request, resource_id): + """ + This function returns the resource detail page. + """ + resource_obj = get_object_or_404(Resource, pk=resource_id) + resource_metadata = _extract_metadata_from_resource(resource_obj) + resource = { + "id": resource_obj.id, + "name": resource_obj.name, + "description": resource_obj.description, + "resource_type": resource_obj.resource_type.name, + "main_resource_category": resource_obj.main_resource_category.name, + "additional_resource_category": ( + resource_obj.additional_resource_category.name + if resource_obj.additional_resource_category + else None + ), + "age_range": resource_obj.age_range, + "curriculum": resource_obj.curriculum, + "pdf_filenames": resource_metadata.pdf_filenames, + "pdf_urls": resource_metadata.pdf_urls, + "thumbnails": list( + zip( + resource_metadata.thumbnail_urls, + resource_metadata.thumbnail_filenames, + strict=False, + ), + ), + "thumbnail_filenames": resource_metadata.thumbnail_filenames, + "thumbnail_urls": resource_metadata.thumbnail_urls, + "snapshot_urls": resource_metadata.snapshot_urls, + "created": resource_metadata.created, + "updated": resource_metadata.updated, + } + return render(request, "resources/resource_detail.html", {"resource": resource}) + + +@login_required +def hx_download_button(request): + """ + This is an HTMX view that is called when the user clicks the download button. + :param + :return: + """ + pdf = request.GET.get("rn") + res = Resource.objects.get(pdf_filename=pdf) + return render( + request, + "resources/hx_download_button.html", + {"pdf_url": _extract_metadata_from_resource(res).pdf_url}, + ) diff --git a/pyblackbird_cc/static/css/project.css b/pyblackbird_cc/static/css/project.css index f1d543d..80ccb8d 100644 --- a/pyblackbird_cc/static/css/project.css +++ b/pyblackbird_cc/static/css/project.css @@ -6,8 +6,48 @@ border-color: #d6e9c6; } +* { + font-family: 'Franklin'; + margin: 0; + padding: 0; +} + .alert-error { color: #b94a48; background-color: #f2dede; border-color: #eed3d7; } + +@font-face { + font-family: 'Franklin'; + src: url(../fonts/LibreFranklin-VariableFont_wght.ttf); +} + +@font-face { + font-family: 'Franklin-Italic'; + src: url(../fonts/LibreFranklin-Italic.ttf); +} + +body { + background-color: #eee; +} + +h1 { + color: #657c76; + font-size: 2.0rem; +} + +h2 { + color: #4e8273; + /* color: white; */ + font-size: 1.8rem; + padding-bottom: 10px; + margin-right: 20px; + margin-top: 20px; + border-radius: 3px 3px 0 0; + padding-left: 2px; + background-size: 100% 50px; + background-repeat: no-repeat; + border-bottom: 10px solid #E4E5E7; + overflow: auto; +} diff --git a/pyblackbird_cc/templates/base.html b/pyblackbird_cc/templates/base.html index 1eb9966..ce0a4ca 100644 --- a/pyblackbird_cc/templates/base.html +++ b/pyblackbird_cc/templates/base.html @@ -18,27 +18,27 @@ {% block extra_css %} {% endblock extra_css %} {% block css %} - <!-- Latest compiled and minified Bootstrap CSS --> - {# <link rel="stylesheet"#} - {# href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.2.3/css/bootstrap.min.css"#} - {# integrity="sha512-SbiR/eusphKoMVVXysTKG/7VseWii+Y3FdHrt0EpKgpToZeemhqHeZeLWLhJutz/2ut2Vw1uQEj2MbRF+TVBUA=="#} - {# crossorigin="anonymous"#} - {# referrerpolicy="no-referrer" />#} + <!--Latest compiled and minified Bootstrap CSS --> + <link rel="stylesheet" + href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.2.3/css/bootstrap.min.css" + integrity="sha512-SbiR/eusphKoMVVXysTKG/7VseWii+Y3FdHrt0EpKgpToZeemhqHeZeLWLhJutz/2ut2Vw1uQEj2MbRF+TVBUA==" + crossorigin="anonymous" + referrerpolicy="no-referrer" /> <!-- Your stuff: Third-party CSS libraries go here --> <!-- This file stores project-specific CSS --> - <!-- <link href="{% static 'css/project.css' %}" rel="stylesheet" /> --> - <link rel="stylesheet" href="{% static "css/wrapper.css" %}" type="text/css" media="screen" charset="utf-8" /> + <link href="{% static 'css/project.css' %}" rel="stylesheet" /> + {# <link rel="stylesheet" href="{% static "css/wrapper.css" %}" type="text/css" media="screen" charset="utf-8" />#} {% endblock css %} <!-- Le javascript - ================================================== --> + ================================================== --> {# Placed at the top of the document so pages load faster with defer #} {% block javascript %} <!-- Bootstrap JS --> - {#<script defer#} - {# src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.2.3/js/bootstrap.min.js"#} - {# integrity="sha512-1/RvZTcCDEUjY/CypiMz+iqqtaoQfAITmNSJY17Myp4Ms5mdxPS5UV7iOfdZoxcGhzFbOm6sntTKJppjvuhg4g=="#} - {# crossorigin="anonymous"#} - {# referrerpolicy="no-referrer"></script>#} + <script defer + src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.2.3/js/bootstrap.min.js" + integrity="sha512-1/RvZTcCDEUjY/CypiMz+iqqtaoQfAITmNSJY17Myp4Ms5mdxPS5UV7iOfdZoxcGhzFbOm6sntTKJppjvuhg4g==" + crossorigin="anonymous" + referrerpolicy="no-referrer"></script> <!-- Your stuff: Third-party javascript libraries go here --> <!-- place project specific Javascript in this file --> <script defer src="{% static 'js/project.js' %}"></script> @@ -47,32 +47,32 @@ </head> <body> <header> - <div class="inner-header"> + <div class="container text-center"> <div> </div> - <div class="info"> - <h1>Joanna Lemon Learning</h1> - <p> - Experienced primary school teacher, dedicated to creating colourful and - clear educational learning resources. - </p> + <div class="row"> + <div class="col"> + <h1>Joanna Lemon Learning</h1> + <p> + Experienced primary school teacher, dedicated to creating colourful and + clear educational learning resources. + </p> + </div> </div> - <div> </div> - <div> </div> - <div class="info"> + <div class="row"> <p> For information: please contact <a href="mailto:joanna@joannalemon.com">joanna@joannalemon.com</a> </p> - <div class="jumplist"> - <div align="center"> - <a href="https://www.tes.com/teaching-resources/shop/joannalemon" - target="_blank">TES Shop</a> - </div> - <div align="center"> - <a href="https://joannalemon.etsy.com/" target="_blank">Etsy</a> - </div> - <div align="center"> - <a href="https://blog.joannalemon.com" target="_blank">Blog</a> - </div> + </div> + <div class="row"> + <div class="col"> + <a href="https://www.tes.com/teaching-resources/shop/joannalemon" + target="_blank">TES Shop</a> + </div> + <div class="col"> + <a href="https://joannalemon.etsy.com/" target="_blank">Etsy</a> + </div> + <div class="col"> + <a href="https://blog.joannalemon.com" target="_blank">Blog</a> </div> </div> </div> diff --git a/pyblackbird_cc/templates/resources/hx_download_button.html b/pyblackbird_cc/templates/resources/hx_download_button.html new file mode 100644 index 0000000..1c956a5 --- /dev/null +++ b/pyblackbird_cc/templates/resources/hx_download_button.html @@ -0,0 +1,3 @@ +<p> + <a href="{{ pdf_url }}">Download the resource</a> +</p> diff --git a/pyblackbird_cc/templates/resources/resource_create.html b/pyblackbird_cc/templates/resources/resource_create.html new file mode 100644 index 0000000..03a192a --- /dev/null +++ b/pyblackbird_cc/templates/resources/resource_create.html @@ -0,0 +1,21 @@ +{% extends "base.html" %} + +{% load static %} +{% load crispy_forms_tags %} + +{# {% block extra_css %}#} +{# <link rel="stylesheet" href="{% static 'css/forms.css' %}">#} +{# {% endblock %}#} +{% block content %} + <div class="container"> + <h2>Upload a new resource</h2> + <p>{% lorem %}</p> + <form action="{% url 'resources:create_resource' %}" + method="post" + enctype="multipart/form-data"> + {% csrf_token %} + {{ form|crispy }} + <input type="submit" class="btn btn-primary" value="Upload" /> + </form> + </div> +{% endblock content %} diff --git a/pyblackbird_cc/templates/resources/resource_detail.html b/pyblackbird_cc/templates/resources/resource_detail.html new file mode 100644 index 0000000..61d9265 --- /dev/null +++ b/pyblackbird_cc/templates/resources/resource_detail.html @@ -0,0 +1,50 @@ +{% extends 'base.html' %} + +{% block content %} + <div class="resource-metadata-panel"> + <div> + <h1 class="resource-title">{{ resource.name }}</h1> + </div> + <div class="resource-details"> + <div>Subject: {{ resource.main_resource_category }}</div> + <div>Age range: {{ resource.age_range }}</div> + <div>Resource type: {{ resource.resource_type }}</div> + </div> + </div> + <div> + <h3 class="feature-pdf-page-title">Feature images</h3> + <div class="resource-img-detail"> + {% for tn_url, tn_filename in resource.thumbnails %}<img src="{{ tn_url }}" alt="{{ tn_filename }}" />{% endfor %} + </div> + <h3 class="feature-pdf-page-title">Resource preview</h3> + <div class="resource-img-detail"> + {% for snapshot_filename, snapshot_urls in resource.snapshot_urls.items %} + <div> + <h4>{{ snapshot_filename }}</h4> + </div> + <div> + {% for snapshot_url in snapshot_urls %}<img src="{{ snapshot_url }}" alt="{{ snapshot_filename }}"{% endfor %} </div /> + {% endfor %} + </div> + <div class="resource-description-panel"> + <h3>What's included?</h3> + <div>{{ resource.description }}</div> + <h3>What's it for?</h3> + <div>{% lorem %}</div> + <h3>Resource Details</h3> + <div>{% lorem %}</div> + </div> + <div class="resource-download-panel"> + <h4>Download the resource</h4> + <div> + Click + <button hx-get="/hx-download-btn?rn={{ resource.pdf_filename }}" + hx-target="next #download-reveal">here</button> + to download the resource + </div> + <div id="download-reveal"></div> + </div> + </div> + </div> + <div>Logged in as {{ request.user.username }}</div> + {% endblock content %} diff --git a/pyblackbird_cc/templates/resources/resource_list.html b/pyblackbird_cc/templates/resources/resource_list.html new file mode 100644 index 0000000..5b11ff6 --- /dev/null +++ b/pyblackbird_cc/templates/resources/resource_list.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} + +{% block title %} + Joanna Lemon Resources - Resource List +{% endblock title %} +{% block content %} + <div class="cella"> + {% if request.user.is_authenticated %} + <div class="admin-links"> + <div> + <a href="{% url 'resources:create_resource' %} ">Add a new resource</a> + </div> + <div> + <form action="{% url 'account_logout' %}" method="post"> + {% csrf_token %} + <button type="submit">Log out</button> + </form> + </div> + </div> + {% endif %} + {% if resource_list %} + {% for resource in resource_list %} + <div class="resource-metadata-panel"> + <div> + <a class="no-underline" + href="{% url 'resources:resource_detail' resource_id=resource.id %}"> + <h1 class="resource-title">{{ resource.name }}</h1> + </a> + </div> + <div> + <div>Main category: {{ resource.main_resource_category_name }}</div> + {% if resource.additional_resource_category_name %} + <div>Additional category: {{ resource.additional_resource_category_name }}</div> + {% endif %} + </div> + </div> + <div> + <div class="resource-img-detail"> + {% for img in resource.thumbnail_urls %} + <img style="max-height: 400px; + width: auto" + src="{{ img }}" + alt="{{ resource.thumbnail_filename }}" /> + {% endfor %} + </div> + <div> + Click + <button hx-get="/hx-download-btn?rn={{ resource.pdf_filename }}" + hx-target="next #download-reveal">here</button> + to download the resource + </div> + <div id="download-reveal"></div> + <hr /> + </div> + {% endfor %} + {% else %} + <p>There are no resources</p> + {% endif %} + </div> + <div>Logged in as {{ request.user.username }}</div> +{% endblock content %} diff --git a/requirements/base.txt b/requirements/base.txt index a7de0ef..573a5cb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ python-slugify==8.0.4 # https://github.com/un33k/python-slugify -Pillow==10.3.0 # https://github.com/python-pillow/Pillow +pillow==10.3.0 # https://github.com/python-pillow/Pillow argon2-cffi==23.1.0 # https://github.com/hynek/argon2_cffi whitenoise==6.6.0 # https://github.com/evansd/whitenoise redis==5.0.4 # https://github.com/redis/redis-py @@ -8,9 +8,17 @@ celery==5.4.0 # pyup: < 6.0 # https://github.com/celery/celery django-celery-beat==2.6.0 # https://github.com/celery/django-celery-beat flower==2.0.1 # https://github.com/mher/flower +# migrated stuff from pyblackbird +pypdf==4.2.0 +pypdfium2==4.29.0 +python-magic==0.4.27 +python-dotenv==1.0.1 +boto3==1.34.89 +sqlparse==0.5.0 + # Django # ------------------------------------------------------------------------------ -django==4.2.13 # pyup: < 5.0 # https://www.djangoproject.com/ +Django==5.0.4 # pyup: < 5.0 # https://www.djangoproject.com/ django-environ==0.11.2 # https://github.com/joke2k/django-environ django-model-utils==4.5.1 # https://github.com/jazzband/django-model-utils django-allauth[mfa]==0.62.1 # https://github.com/pennersr/django-allauth |