From 46287d698487b0659342a47119f00ed72a3416cf Mon Sep 17 00:00:00 2001 From: Mihkel Martin Kasterpalu Date: Tue, 11 Feb 2025 15:50:27 +0200 Subject: [PATCH] Optimize rahvatarkus API, DB schema and data loading --- src/lib/server/db/schema.ts | 30 +++++- src/routes/api/rahvatarkus/answer/+server.ts | 78 +++++++-------- .../archive/[limit]/[[offset]]/+server.ts | 56 +++++------ .../api/rahvatarkus/question/+server.ts | 94 ++++++++++++++----- .../vinge/rahvatarkus/+layout.server.ts | 43 +++++++++ src/routes/vinge/rahvatarkus/+layout.svelte | 13 +++ src/routes/vinge/rahvatarkus/+page.server.ts | 41 ++------ src/routes/vinge/rahvatarkus/+page.svelte | 16 ++-- .../vinge/rahvatarkus/answer-form.svelte | 1 + .../vinge/rahvatarkus/question-form.svelte | 2 + 10 files changed, 227 insertions(+), 147 deletions(-) create mode 100644 src/routes/vinge/rahvatarkus/+layout.server.ts create mode 100644 src/routes/vinge/rahvatarkus/+layout.svelte diff --git a/src/lib/server/db/schema.ts b/src/lib/server/db/schema.ts index 0da2640..71f6aab 100644 --- a/src/lib/server/db/schema.ts +++ b/src/lib/server/db/schema.ts @@ -1,24 +1,44 @@ import { relations } from 'drizzle-orm/relations'; -import { sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core'; import { nanoid } from 'nanoid'; export const questions = sqliteTable('questions', { id: text('id') .primaryKey() .$defaultFn(() => nanoid()), - creator: text('creator').notNull(), // session token - content: text('content').notNull().unique() + creator: text('creator').notNull(), + content: text('content').notNull().unique(), + answerCount: integer('answer_count').notNull().default(0), + createdAt: integer('created_at', { mode: 'timestamp' }) + .notNull() + .$defaultFn(() => new Date()) }); export const answers = sqliteTable('answers', { id: text('id') .primaryKey() .$defaultFn(() => nanoid()), - creator: text('creator').notNull(), // session token + creator: text('creator').notNull(), content: text('content').notNull(), - questionId: text('question_id').references(() => questions.id) + questionId: text('question_id') + .notNull() + .references(() => questions.id), + createdAt: integer('created_at', { mode: 'timestamp' }) + .notNull() + .$defaultFn(() => new Date()) }); +// Indexes +export const questionsIndexes = { + answerCountIdx: `CREATE INDEX idx_questions_answer_count ON questions (answer_count)`, + creatorIdx: `CREATE INDEX idx_questions_creator ON questions (creator)` +}; + +export const answersIndexes = { + questionIdIdx: `CREATE INDEX idx_answers_question_id ON answers (question_id)`, + creatorIdx: `CREATE INDEX idx_answers_creator ON answers (creator)` +}; + // Relations export const questionsRelations = relations(questions, ({ many }) => ({ answers: many(answers) diff --git a/src/routes/api/rahvatarkus/answer/+server.ts b/src/routes/api/rahvatarkus/answer/+server.ts index 6e1d684..b5cae15 100644 --- a/src/routes/api/rahvatarkus/answer/+server.ts +++ b/src/routes/api/rahvatarkus/answer/+server.ts @@ -1,72 +1,62 @@ import { json } from '@sveltejs/kit'; import { db } from '$lib/server/db'; import { questions, answers } from '$lib/server/db/schema'; -import { eq } from 'drizzle-orm'; +import { eq, sql } from 'drizzle-orm'; const maxAnswers = 5; export async function POST({ locals, request }) { - const { userId, questionId, content }: { userId: string; questionId: string; content: string } = - await request.json(); + const { userId, questionId, content } = await request.json(); const { session } = locals; - if (!session?.data?.userId) { - return; - } - + if (!session?.data?.userId) return; const user = session.data.userId; if (!user || !userId || user !== userId) { return json({ error: 'Unauthorized' }, { status: 401 }); } - if (!content) { - return json({ error: 'Answer is required' }, { status: 400 }); - } + // Start transaction + return await db.transaction(async (tx) => { + // Get question and validate in one query + const question = await tx + .select({ + creator: questions.creator, + answerCount: questions.answerCount + }) + .from(questions) + .where(eq(questions.id, questionId)) + .limit(1); - if (!questionId) { - return json({ error: 'No question specified' }, { status: 400 }); - } + if (!question.length) { + return json({ error: 'Question not found' }, { status: 400 }); + } - const question = await db - .select({ - question: questions - }) - .from(questions) - .where(eq(questions.id, questionId)) - .limit(1); + const [questionData] = question; - if (!question) { - return json({ error: 'Question not found' }, { status: 400 }); - } + // if (questionData.creator === user) { + // return json({ error: 'Cannot answer own question' }, { status: 400 }); + // } - if (question[0].question.creator === user) { - return json({ error: 'Not allowed to answer your own question' }, { status: 400 }); - } + if (questionData.answerCount >= maxAnswers) { + return json({ error: 'No more answers needed' }, { status: 400 }); + } - const currentAnswers = await db - .select({ - answer: answers - }) - .from(answers) - .where(eq(answers.questionId, questionId)); - - if (currentAnswers.length >= maxAnswers) { - return json({ error: 'No more answers needed for this question' }, { status: 400 }); - } - - try { - const [newAnswer] = await db + // Insert answer and update count atomically + const [newAnswer] = await tx .insert(answers) .values({ - content: content, + content, creator: userId, - questionId: questionId + questionId }) .returning(); + await tx + .update(questions) + .set({ answerCount: sql`${questions.answerCount} + 1` }) + .where(eq(questions.id, questionId)); + return json(newAnswer); - } catch (e) { - return json({ error: e }, { status: 400 }); - } + }); } diff --git a/src/routes/api/rahvatarkus/archive/[limit]/[[offset]]/+server.ts b/src/routes/api/rahvatarkus/archive/[limit]/[[offset]]/+server.ts index 94b25f8..df43a98 100644 --- a/src/routes/api/rahvatarkus/archive/[limit]/[[offset]]/+server.ts +++ b/src/routes/api/rahvatarkus/archive/[limit]/[[offset]]/+server.ts @@ -1,52 +1,48 @@ import { json } from '@sveltejs/kit'; import { db } from '$lib/server/db'; import { questions, answers } from '$lib/server/db/schema'; -import { eq, sql } from 'drizzle-orm'; +import { eq, gt, sql } from 'drizzle-orm'; import type { Question } from '$lib/types'; export async function GET({ params }) { const limit = Math.min(parseInt(params.limit) || 10, 10); const offset = parseInt(params.offset) || 0; - // Get total count - const [{ total }] = await db + // Get total in parallel with data + const totalPromise = db .select({ - total: sql`count(DISTINCT ${questions.id})` + count: sql`count(*)` }) .from(questions) - .innerJoin(answers, eq(questions.id, answers.questionId)); + .where(gt(questions.answerCount, 0)); - const results = await db + const questionsPromise = db .select({ - question: questions, - answers: answers + id: questions.id, + content: questions.content, + creator: questions.creator, + createdAt: questions.createdAt, + answers: sql`json_group_array(json_object( + 'id', ${answers.id}, + 'content', ${answers.content}, + 'creator', ${answers.creator}, + 'createdAt', ${answers.createdAt} + ))` }) .from(questions) .innerJoin(answers, eq(questions.id, answers.questionId)) + .where(gt(questions.answerCount, 0)) + .groupBy(questions.id) .limit(limit) .offset(offset); - // Group answers by question - type QuestionMap = { - [key: string]: Question; - }; + const [total, curr_questions] = await Promise.all([totalPromise, questionsPromise]); - const questionsWithAnswers = results.reduce((acc, row) => { - const questionId = row.question.id; - - if (!acc[questionId]) { - acc[questionId] = { - ...row.question, - answers: [] - }; - } - - if (row.answers) { - acc[questionId].answers.push(row.answers); - } - - return acc; - }, {}); - - return json({ data: Object.values(questionsWithAnswers), meta: { limit, offset, total } }); + return json({ + data: curr_questions.map((q) => ({ + ...q, + answers: JSON.parse(q.answers) + })), + meta: { limit, offset, total: total[0].count } + }); } diff --git a/src/routes/api/rahvatarkus/question/+server.ts b/src/routes/api/rahvatarkus/question/+server.ts index f9b69a9..f90e6b8 100644 --- a/src/routes/api/rahvatarkus/question/+server.ts +++ b/src/routes/api/rahvatarkus/question/+server.ts @@ -1,67 +1,109 @@ import { json } from '@sveltejs/kit'; import { db } from '$lib/server/db'; import { questions, answers } from '$lib/server/db/schema'; -import { eq, count, and, not, sql } from 'drizzle-orm'; +import { eq, and, not, sql, exists, lt, gt } from 'drizzle-orm'; export async function GET({ locals }) { const { session } = locals; - - if (!session?.data?.userId) { - return; - } + if (!session?.data?.userId) return; const user = session.data.userId; - const questionWithAnswerCount = await db + // Use the answerCount field and avoid joins + const eligibleQuestions = await db .select({ id: questions.id, content: questions.content, - answerCount: count(answers.id) + answerCount: questions.answerCount }) .from(questions) - .leftJoin(answers, eq(questions.id, answers.questionId)) - .groupBy(questions.id) - .having(and(sql`${count(answers.id)} < 5`, not(eq(questions.creator, user)))) + .where( + and( + lt(questions.answerCount, 5), + not( + exists( + db + .select() + .from(answers) + .where(and(eq(answers.questionId, questions.id), eq(answers.creator, user))) + ) + ) + ) + ) .orderBy(sql`RANDOM()`) .limit(1); - if (!questionWithAnswerCount.length) { + if (!eligibleQuestions.length) { return json({ error: 'No questions available' }, { status: 404 }); } - return json(questionWithAnswerCount[0]); + return json(eligibleQuestions[0]); } export async function POST({ locals, request }) { const { userId, content }: { userId: string; content: string } = await request.json(); const { session } = locals; - if (!session?.data?.userId) { - return; - } - + if (!session?.data?.userId) return; const user = session.data.userId; if (!user || !userId || user !== userId) { - console.log(user, userId); return json({ error: 'Unauthorized' }, { status: 401 }); } - if (!content) { + if (!content?.trim()) { return json({ error: 'Content is required' }, { status: 400 }); } + // Normalize content + const normalizedContent = content.trim(); + const finalContent = + normalizedContent.at(-1) === '?' ? normalizedContent.slice(0, -1) : normalizedContent; + try { - const [newQuestion] = await db - .insert(questions) - .values({ - content: content.at(-1) === '?' ? content.slice(0, -1) : content, - creator: userId - }) - .returning(); + // Use transaction to ensure data consistency + const [newQuestion] = await db.transaction(async (tx) => { + // Check for duplicate questions first + const existingQuestion = await tx + .select({ id: questions.id }) + .from(questions) + .where(eq(questions.content, finalContent)) + .limit(1); + + if (existingQuestion.length > 0) { + throw new Error('Question already exists'); + } + + // Check user's recent questions (optional rate limiting) + const recentQuestions = await tx + .select({ count: sql`count(*)` }) + .from(questions) + .where( + and( + eq(questions.creator, userId), + gt(questions.createdAt, sql`datetime('now', '-1 hour')`) + ) + ); + + if (recentQuestions[0].count >= 10) { + throw new Error('Too many questions in the last hour'); + } + + // Insert the new question + return await tx + .insert(questions) + .values({ + content: finalContent, + creator: userId, + answerCount: 0, + createdAt: new Date() + }) + .returning(); + }); return json(newQuestion); } catch (e) { - return json({ error: e }, { status: 400 }); + const error = e instanceof Error ? e.message : 'Failed to create question'; + return json({ error }, { status: 400 }); } } diff --git a/src/routes/vinge/rahvatarkus/+layout.server.ts b/src/routes/vinge/rahvatarkus/+layout.server.ts new file mode 100644 index 0000000..29f066d --- /dev/null +++ b/src/routes/vinge/rahvatarkus/+layout.server.ts @@ -0,0 +1,43 @@ +import type { LayoutServerLoad } from './$types'; +import type { Question } from '$lib/types'; +import { formSchema as questionSchema } from './question-schema'; +import { formSchema as answerSchema } from './answer-schema'; + +import { superValidate } from 'sveltekit-superforms'; +import { zod } from 'sveltekit-superforms/adapters'; +import { nanoid } from 'nanoid'; + +export const load: LayoutServerLoad = async ({ fetch, locals }) => { + const { session } = locals; + + if (!session?.data?.userId) { + await session.setData({ userId: nanoid() }); + await session.save(); + } + + const user = session.data.userId; + + const res = await fetch('/api/rahvatarkus/question') + .then(async (res) => { + const data = await res.json(); + return { ok: res.ok, data: data }; + }) + .then((data) => { + return data; + }); + + let question: Question | undefined = undefined; + + if (res.ok) { + question = res.data; + } + + return { + user: user, + question: question, + question_form: await superValidate(zod(questionSchema)), + answer_form: await superValidate({ questionId: question?.id }, zod(answerSchema), { + errors: false + }) + }; +}; diff --git a/src/routes/vinge/rahvatarkus/+layout.svelte b/src/routes/vinge/rahvatarkus/+layout.svelte new file mode 100644 index 0000000..5c8d3b8 --- /dev/null +++ b/src/routes/vinge/rahvatarkus/+layout.svelte @@ -0,0 +1,13 @@ + + + + + +{@render children()} diff --git a/src/routes/vinge/rahvatarkus/+page.server.ts b/src/routes/vinge/rahvatarkus/+page.server.ts index 148561b..7b0cabe 100644 --- a/src/routes/vinge/rahvatarkus/+page.server.ts +++ b/src/routes/vinge/rahvatarkus/+page.server.ts @@ -1,41 +1,17 @@ import type { Actions, PageServerLoad } from './$types'; -import type { Question } from '$lib/types'; import { formSchema as questionSchema } from './question-schema'; import { formSchema as answerSchema } from './answer-schema'; import { superValidate } from 'sveltekit-superforms'; import { zod } from 'sveltekit-superforms/adapters'; -import { nanoid } from 'nanoid'; import { fail } from '@sveltejs/kit'; -const perPage = 5; +const pageSize = 5; -export const load: PageServerLoad = async ({ fetch, locals }) => { - const { session } = locals; +export const load: PageServerLoad = async ({ fetch, url }) => { + const page = Number(url.searchParams.get('leht')) || 0; - if (!session?.data?.userId) { - await session.setData({ userId: nanoid() }); - await session.save(); - } - - const user = session.data.userId; - - const res = await fetch('/api/rahvatarkus/question') - .then(async (res) => { - const data = await res.json(); - return { ok: res.ok, data: data }; - }) - .then((data) => { - return data; - }); - - let question: Question | undefined = undefined; - - if (res.ok) { - question = res.data; - } - - const archiveRes = fetch(`/api/rahvatarkus/archive/${perPage}`) + const archiveRes = fetch(`/api/rahvatarkus/archive/${pageSize}/${(page - 1) * pageSize}`) .then((res) => { return res.json(); }) @@ -44,13 +20,8 @@ export const load: PageServerLoad = async ({ fetch, locals }) => { }); return { - user: user, - question: question, - question_form: await superValidate(zod(questionSchema)), - answer_form: await superValidate({ questionId: question?.id }, zod(answerSchema), { - errors: false - }), - perPage, + page, + pageSize, streamed: { archive: archiveRes } diff --git a/src/routes/vinge/rahvatarkus/+page.svelte b/src/routes/vinge/rahvatarkus/+page.svelte index f18ea7d..9ad0fc3 100644 --- a/src/routes/vinge/rahvatarkus/+page.svelte +++ b/src/routes/vinge/rahvatarkus/+page.svelte @@ -4,18 +4,13 @@ import * as Accordion from '$lib/components/ui/accordion/index.js'; import * as Pagination from '$lib/components/ui/pagination/index.js'; - import QuestionForm from './question-form.svelte'; - import AnswerForm from './answer-form.svelte'; - import SuperDebug from 'sveltekit-superforms'; + import { goto } from '$app/navigation'; let { data }: { data: PageData } = $props(); $inspect(data); - - - {#await data.streamed.archive}

loading

{:then archive} @@ -35,7 +30,14 @@ {/each} - + { + goto(`?leht=${value}`); + }} + count={archive.meta.total} + perPage={data.pageSize} + page={data.page} + > {#snippet children({ pages, currentPage })} diff --git a/src/routes/vinge/rahvatarkus/answer-form.svelte b/src/routes/vinge/rahvatarkus/answer-form.svelte index b0d5195..82bfdb1 100644 --- a/src/routes/vinge/rahvatarkus/answer-form.svelte +++ b/src/routes/vinge/rahvatarkus/answer-form.svelte @@ -14,6 +14,7 @@ const form = superForm(data.answer_form, { validators: zodClient(formSchema), + invalidateAll: 'force', onUpdated: ({ form: f }) => { if (f.valid) { toast.success(`You submitted ${JSON.stringify(f.data, null, 2)}`); diff --git a/src/routes/vinge/rahvatarkus/question-form.svelte b/src/routes/vinge/rahvatarkus/question-form.svelte index 3c2c25c..780a555 100644 --- a/src/routes/vinge/rahvatarkus/question-form.svelte +++ b/src/routes/vinge/rahvatarkus/question-form.svelte @@ -12,6 +12,8 @@ const form = superForm(data.question_form, { validators: zodClient(formSchema), + invalidateAll: false, + resetForm: true, onUpdated: ({ form: f }) => { if (f.valid) { toast.success(`You submitted ${JSON.stringify(f.data, null, 2)}`);