diff --git a/go.mod b/go.mod index 4178f45..846e1e6 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( github.com/gofiber/template/v2 v2.1.0 // indirect github.com/gofiber/utils/v2 v2.0.2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gopherlibs/feedhub v1.2.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/go.sum b/go.sum index 0211568..c1a1202 100644 --- a/go.sum +++ b/go.sum @@ -279,6 +279,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherlibs/feedhub v1.2.0 h1:1nfM8gRoiA+VNjKc1FzrwiXkrBKsnAghA3PVvgAiSI0= +github.com/gopherlibs/feedhub v1.2.0/go.mod h1:vvQEZzTKr2KhO0mCdEUGfKLvUJFfO8U+WUpaMyoZttc= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= diff --git a/layouts/simplecss/layout_main.html b/layouts/simplecss/layout_main.html index 5c4203e..cc2e616 100644 --- a/layouts/simplecss/layout_main.html +++ b/layouts/simplecss/layout_main.html @@ -4,6 +4,8 @@ {{ .Site.Title }} + + diff --git a/models/iters.go b/models/iters.go new file mode 100644 index 0000000..8d9d52a --- /dev/null +++ b/models/iters.go @@ -0,0 +1,10 @@ +package models + +type Maybe[T any] struct { + Value T + Err error +} + +func (m Maybe[T]) Get() (T, error) { + return m.Value, m.Err +} diff --git a/models/pubmodel/sites.go b/models/pubmodel/sites.go index 6fc3942..a745885 100644 --- a/models/pubmodel/sites.go +++ b/models/pubmodel/sites.go @@ -1,7 +1,9 @@ package pubmodel import ( + "context" "io" + "iter" "lmika.dev/lmika/weiro/models" ) @@ -9,8 +11,11 @@ import ( type Site struct { models.Site BaseURL string - Posts []*models.Post + //Posts []*models.Post Uploads []models.Upload OpenUpload func(u models.Upload) (io.ReadCloser, error) + + // PostItr returns a new post iterator + PostIter func(ctx context.Context) iter.Seq[models.Maybe[*models.Post]] } diff --git a/providers/db/gen/sqlgen/posts.sql.go b/providers/db/gen/sqlgen/posts.sql.go index e4b00f0..d512941 100644 --- a/providers/db/gen/sqlgen/posts.sql.go +++ b/providers/db/gen/sqlgen/posts.sql.go @@ -123,21 +123,28 @@ func (q *Queries) SelectPostByGUID(ctx context.Context, guid string) (Post, erro const selectPostsOfSite = `-- name: SelectPostsOfSite :many SELECT id, site_id, state, guid, title, body, slug, created_at, updated_at, published_at, deleted_at FROM posts -WHERE site_id = ? AND ( +WHERE site_id = ?1 AND ( CASE CAST (?2 AS TEXT) WHEN 'deleted' THEN deleted_at > 0 ELSE deleted_at = 0 END -) ORDER BY created_at DESC LIMIT 10 +) ORDER BY created_at DESC LIMIT ?4 OFFSET ?3 ` type SelectPostsOfSiteParams struct { SiteID int64 PostFilter string + Offset int64 + Limit int64 } func (q *Queries) SelectPostsOfSite(ctx context.Context, arg SelectPostsOfSiteParams) ([]Post, error) { - rows, err := q.db.QueryContext(ctx, selectPostsOfSite, arg.SiteID, arg.PostFilter) + rows, err := q.db.QueryContext(ctx, selectPostsOfSite, + arg.SiteID, + arg.PostFilter, + arg.Offset, + arg.Limit, + ) if err != nil { return nil, err } diff --git a/providers/db/posts.go b/providers/db/posts.go index 04a7a3e..218e931 100644 --- a/providers/db/posts.go +++ b/providers/db/posts.go @@ -8,7 +8,12 @@ import ( "lmika.dev/lmika/weiro/providers/db/gen/sqlgen" ) -func (db *Provider) SelectPostsOfSite(ctx context.Context, siteID int64, showDeleted bool) ([]*models.Post, error) { +type PagingParams struct { + Limit int64 + Offset int64 +} + +func (db *Provider) SelectPostsOfSite(ctx context.Context, siteID int64, showDeleted bool, pp PagingParams) ([]*models.Post, error) { var filter = "" if showDeleted { filter = "deleted" @@ -17,6 +22,8 @@ func (db *Provider) SelectPostsOfSite(ctx context.Context, siteID int64, showDel rows, err := db.queries.SelectPostsOfSite(ctx, sqlgen.SelectPostsOfSiteParams{ SiteID: siteID, PostFilter: filter, + Limit: pp.Limit, + Offset: pp.Offset, }) if err != nil { return nil, err diff --git a/providers/sitebuilder/builder.go b/providers/sitebuilder/builder.go index acbb62b..16d4fa6 100644 --- a/providers/sitebuilder/builder.go +++ b/providers/sitebuilder/builder.go @@ -6,11 +6,13 @@ import ( "fmt" "html/template" "io" + "iter" "os" "path/filepath" - "sort" "strings" + "time" + "github.com/gopherlibs/feedhub/feedhub" "golang.org/x/sync/errgroup" "lmika.dev/lmika/weiro/models" "lmika.dev/lmika/weiro/models/pubmodel" @@ -47,10 +49,14 @@ func (b *Builder) BuildSite(outDir string) error { return err } - eg := errgroup.Group{} + eg, ctx := errgroup.WithContext(context.Background()) eg.Go(func() error { - for _, post := range b.site.Posts { + for mp := range b.site.PostIter(ctx) { + post, err := mp.Get() + if err != nil { + return err + } if err := b.writePost(buildCtx, post); err != nil { return err } @@ -59,7 +65,14 @@ func (b *Builder) BuildSite(outDir string) error { }) eg.Go(func() error { - if err := b.renderPostList(buildCtx, b.site.Posts); err != nil { + if err := b.renderPostList(buildCtx, b.site.PostIter(ctx)); err != nil { + return err + } + return nil + }) + + eg.Go(func() error { + if err := b.renderFeeds(buildCtx, b.site.PostIter(ctx)); err != nil { return err } return nil @@ -79,14 +92,16 @@ func (b *Builder) BuildSite(outDir string) error { return nil } -func (b *Builder) renderPostList(ctx buildContext, postList []*models.Post) error { +func (b *Builder) renderPostList(ctx buildContext, postIter iter.Seq[models.Maybe[*models.Post]]) error { // TODO: paging - postCopy := make([]*models.Post, len(postList)) - copy(postCopy, postList) - - sort.Slice(postCopy, func(i, j int) bool { - return postCopy[i].PublishedAt.After(postCopy[j].PublishedAt) - }) + postCopy := make([]*models.Post, 0) + for mp := range postIter { + post, err := mp.Get() + if err != nil { + return err + } + postCopy = append(postCopy, post) + } pl := postListData{ commonData: commonData{Site: b.site}, @@ -104,6 +119,68 @@ func (b *Builder) renderPostList(ctx buildContext, postList []*models.Post) erro }) } +func (b *Builder) renderFeeds(ctx buildContext, postIter iter.Seq[models.Maybe[*models.Post]]) error { + now := time.Now() + feed := &feedhub.Feed{ + Title: b.site.Title, + Link: &feedhub.Link{Href: b.site.BaseURL}, + Description: b.site.Tagline, + // TO FIX: Author + Created: now, + } + + items := 0 + for mp := range postIter { + post, err := mp.Get() + if err != nil { + return fmt.Errorf("failed to get post: %w", err) + } + + renderedPost, err := b.renderPost(post) + if err != nil { + return err + } + + feed.Items = append(feed.Items, &feedhub.Item{ + Id: filepath.Join(b.site.BaseURL, post.GUID), + Title: post.Title, + Link: &feedhub.Link{Href: renderedPost.PostURL}, + Content: string(renderedPost.HTML), + // TO FIX: Created should be first published + Created: post.PublishedAt, + Updated: post.PublishedAt, + }) + + items++ + if items >= b.opts.FeedItems { + break + } + } + + if err := b.createAtPath(ctx, "/feed.xml", func(f io.Writer) error { + rss, err := feed.ToRss() + if err != nil { + return fmt.Errorf("failed to convert feed to RSS: %w", err) + } + _, err = io.WriteString(f, rss) + return err + }); err != nil { + return err + } + + if err := b.createAtPath(ctx, "/feed.json", func(f io.Writer) error { + rss, err := feed.ToJSON() + if err != nil { + return fmt.Errorf("failed to convert feed to JSON feed: %w", err) + } + _, err = io.WriteString(f, rss) + return err + }); err != nil { + return err + } + return nil +} + func (b *Builder) renderPost(post *models.Post) (postSingleData, error) { postPath := post.Slug if b.opts.BasePosts != "" { @@ -115,10 +192,13 @@ func (b *Builder) renderPost(post *models.Post) (postSingleData, error) { return postSingleData{}, fmt.Errorf("failed to write post %s: %w", post.Slug, err) } + postURL := strings.TrimSuffix(b.site.BaseURL, "/") + "/" + strings.TrimPrefix(postPath, "/") + return postSingleData{ commonData: commonData{Site: b.site}, Path: postPath, Post: post, + PostURL: postURL, HTML: template.HTML(md.String()), }, nil } diff --git a/providers/sitebuilder/tmpls.go b/providers/sitebuilder/tmpls.go index 812a14d..fa70b6d 100644 --- a/providers/sitebuilder/tmpls.go +++ b/providers/sitebuilder/tmpls.go @@ -29,6 +29,9 @@ type Options struct { // TemplatesFS provides the raw templates for rendering the site. TemplatesFS fs.FS + // FeedItems holds the number of posts to show in the feed. + FeedItems int + RenderTZ *time.Location } @@ -38,9 +41,10 @@ type commonData struct { type postSingleData struct { commonData - Post *models.Post - HTML template.HTML - Path string + Post *models.Post + HTML template.HTML + Path string + PostURL string } type postListData struct { diff --git a/services/posts/list.go b/services/posts/list.go index e94b24b..ae70e1c 100644 --- a/services/posts/list.go +++ b/services/posts/list.go @@ -4,6 +4,7 @@ import ( "context" "lmika.dev/lmika/weiro/models" + "lmika.dev/lmika/weiro/providers/db" ) func (s *Service) ListPosts(ctx context.Context, showDeleted bool) ([]*models.Post, error) { @@ -12,7 +13,10 @@ func (s *Service) ListPosts(ctx context.Context, showDeleted bool) ([]*models.Po return nil, models.SiteRequiredError } - posts, err := s.db.SelectPostsOfSite(ctx, site.ID, showDeleted) + posts, err := s.db.SelectPostsOfSite(ctx, site.ID, showDeleted, db.PagingParams{ + Offset: 0, + Limit: 25, + }) if err != nil { return nil, err } diff --git a/services/publisher/iter.go b/services/publisher/iter.go new file mode 100644 index 0000000..a125fb1 --- /dev/null +++ b/services/publisher/iter.go @@ -0,0 +1,37 @@ +package publisher + +import ( + "context" + "iter" + + "lmika.dev/lmika/weiro/models" + "lmika.dev/lmika/weiro/providers/db" +) + +// PostIter returns a post iterator which returns posts in reverse chronological order. +func (s *Publisher) postIter(ctx context.Context, site int64) iter.Seq[models.Maybe[*models.Post]] { + return func(yield func(models.Maybe[*models.Post]) bool) { + paging := db.PagingParams{Offset: 0, Limit: 50} + page, err := s.db.SelectPostsOfSite(ctx, site, false, paging) + if err != nil { + yield(models.Maybe[*models.Post]{Err: err}) + return + } + + for { + for _, post := range page { + if !yield(models.Maybe[*models.Post]{Value: post}) { + return + } + } + paging.Offset += paging.Limit + page, err = s.db.SelectPostsOfSite(ctx, site, false, paging) + if err != nil { + yield(models.Maybe[*models.Post]{Err: err}) + return + } else if len(page) == 0 { + return + } + } + } +} diff --git a/services/publisher/service.go b/services/publisher/service.go index 7026bca..ec9834e 100644 --- a/services/publisher/service.go +++ b/services/publisher/service.go @@ -3,6 +3,7 @@ package publisher import ( "context" "io" + "iter" "log" "os" @@ -38,12 +39,6 @@ func (p *Publisher) Publish(ctx context.Context, site models.Site) error { return err } - // Fetch all content of site - posts, err := p.db.SelectPostsOfSite(ctx, site.ID, false) - if err != nil { - return err - } - // Fetch all uploads of site uploads, err := p.db.SelectUploadsOfSite(ctx, site.ID) if err != nil { @@ -56,8 +51,10 @@ func (p *Publisher) Publish(ctx context.Context, site models.Site) error { } pubSite := pubmodel.Site{ - Site: site, - Posts: posts, + Site: site, + PostIter: func(ctx context.Context) iter.Seq[models.Maybe[*models.Post]] { + return p.postIter(ctx, site.ID) + }, BaseURL: target.BaseURL, Uploads: uploads, OpenUpload: func(u models.Upload) (io.ReadCloser, error) { @@ -77,6 +74,7 @@ func (p *Publisher) publishSite(ctx context.Context, pubSite pubmodel.Site, targ sb, err := sitebuilder.New(pubSite, sitebuilder.Options{ BasePosts: "/posts", TemplatesFS: simplecss.FS, + FeedItems: 30, }) if err != nil { return err @@ -88,7 +86,11 @@ func (p *Publisher) publishSite(ctx context.Context, pubSite pubmodel.Site, targ if err := exporter.WriteSiteYAML(); err != nil { return err } - for _, p := range pubSite.Posts { + for mp := range pubSite.PostIter(ctx) { + p, err := mp.Get() + if err != nil { + return err + } if err := exporter.WritePost(p); err != nil { return err } diff --git a/sql/queries/posts.sql b/sql/queries/posts.sql index 3f740bf..dae1f39 100644 --- a/sql/queries/posts.sql +++ b/sql/queries/posts.sql @@ -1,12 +1,12 @@ -- name: SelectPostsOfSite :many SELECT * FROM posts -WHERE site_id = ? AND ( +WHERE site_id = sqlc.arg(site_id) AND ( CASE CAST (sqlc.arg(post_filter) AS TEXT) WHEN 'deleted' THEN deleted_at > 0 ELSE deleted_at = 0 END -) ORDER BY created_at DESC LIMIT 10; +) ORDER BY created_at DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); -- name: SelectPost :one SELECT * FROM posts WHERE id = ? LIMIT 1;