Sanitizes posts on publish and render (#217)

* Sanatizes posts on publish and render * lint * lint and added sanitize to thread view as well * adjusts indices based on replaced text * Woops, fixes a bug * bugfix + cleanup * comment * lint * move sanitize text to later in the flow * undo changes to compose post * Add RichText library building upon the sanitizePost library method * Add lodash.clonedeep dep * Switch to RichText processing on record load & render * Fix lint --------- Co-authored-by: Paul Frazee <pfrazee@gmail.com>
2023-02-20 15:09:08 -05:00
parent af1396e396
commit 1861f95a1f
15 changed files with 556 additions and 31 deletions
@@ -0,0 +1,123 @@
+import {AppBskyFeedPost} from '@atproto/api'
+type Entity = AppBskyFeedPost.Entity
+import {RichText} from '../../../src/lib/strings/rich-text'
+import {removeExcessNewlines} from '../../../src/lib/strings/rich-text-sanitize'
+
+describe('removeExcessNewlines', () => {
+  it('removes more than two consecutive new lines', () => {
+    const input = new RichText(
+      'test\n\n\n\n\ntest\n\n\n\n\n\n\ntest\n\n\n\n\n\n\ntest\n\n\n\n\n\n\ntest',
+    )
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual('test\n\ntest\n\ntest\n\ntest\n\ntest')
+  })
+
+  it('removes more than two consecutive new lines with spaces', () => {
+    const input = new RichText(
+      'test\n\n\n\n\ntest\n \n \n \n \n\n\ntest\n\n\n\n\n\n\ntest\n\n\n\n\n  \n\ntest',
+    )
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual('test\n\ntest\n\ntest\n\ntest\n\ntest')
+  })
+
+  it('returns original string if there are no consecutive new lines', () => {
+    const input = new RichText('test\n\ntest\n\ntest\n\ntest\n\ntest')
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual(input.text)
+  })
+
+  it('returns original string if there are no new lines', () => {
+    const input = new RichText('test test          test test test')
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual(input.text)
+  })
+
+  it('returns empty string if input is empty', () => {
+    const input = new RichText('')
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual('')
+  })
+
+  it('works with different types of new line characters', () => {
+    const input = new RichText(
+      'test\r\ntest\n\rtest\rtest\n\n\n\ntest\n\r \n \n \n \n\n\ntest',
+    )
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual('test\r\ntest\n\rtest\rtest\n\ntest\n\ntest')
+  })
+
+  it('removes more than two consecutive new lines with zero width space', () => {
+    const input = new RichText(
+      'test\n\n\n\n\ntest\n\u200B\u200B\n\n\n\ntest\n \u200B\u200B \n\n\n\ntest\n\n\n\n\n\n\ntest',
+    )
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual('test\n\ntest\n\ntest\n\ntest\n\ntest')
+  })
+
+  it('removes more than two consecutive new lines with zero width non-joiner', () => {
+    const input = new RichText(
+      'test\n\n\n\n\ntest\n\u200C\u200C\n\n\n\ntest\n \u200C\u200C \n\n\n\ntest\n\n\n\n\n\n\ntest',
+    )
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual('test\n\ntest\n\ntest\n\ntest\n\ntest')
+  })
+
+  it('removes more than two consecutive new lines with zero width joiner', () => {
+    const input = new RichText(
+      'test\n\n\n\n\ntest\n\u200D\u200D\n\n\n\ntest\n \u200D\u200D \n\n\n\ntest\n\n\n\n\n\n\ntest',
+    )
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual('test\n\ntest\n\ntest\n\ntest\n\ntest')
+  })
+
+  it('removes more than two consecutive new lines with soft hyphen', () => {
+    const input = new RichText(
+      'test\n\n\n\n\ntest\n\u00AD\u00AD\n\n\n\ntest\n \u00AD\u00AD \n\n\n\ntest\n\n\n\n\n\n\ntest',
+    )
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual('test\n\ntest\n\ntest\n\ntest\n\ntest')
+  })
+
+  it('removes more than two consecutive new lines with word joiner', () => {
+    const input = new RichText(
+      'test\n\n\n\n\ntest\n\u2060\u2060\n\n\n\ntest\n \u2060\u2060 \n\n\n\ntest\n\n\n\n\n\n\ntest',
+    )
+    const output = removeExcessNewlines(input)
+    expect(output.text).toEqual('test\n\ntest\n\ntest\n\ntest\n\ntest')
+  })
+})
+
+describe('removeExcessNewlines w/entities', () => {
+  it('preserves entities as expected', () => {
+    const input = new RichText(
+      'test\n\n\n\n\ntest\n\n\n\n\n\n\ntest\n\n\n\n\n\n\ntest\n\n\n\n\n\n\ntest',
+      [
+        {index: {start: 0, end: 13}, type: '', value: ''},
+        {index: {start: 13, end: 24}, type: '', value: ''},
+        {index: {start: 9, end: 15}, type: '', value: ''},
+        {index: {start: 4, end: 9}, type: '', value: ''},
+      ],
+    )
+    const output = removeExcessNewlines(input)
+    expect(entToStr(input.text, input.entities?.[0])).toEqual(
+      'test\n\n\n\n\ntest',
+    )
+    expect(entToStr(input.text, input.entities?.[1])).toEqual(
+      '\n\n\n\n\n\n\ntest',
+    )
+    expect(entToStr(input.text, input.entities?.[2])).toEqual('test\n\n')
+    expect(entToStr(input.text, input.entities?.[3])).toEqual('\n\n\n\n\n')
+    expect(output.text).toEqual('test\n\ntest\n\ntest\n\ntest\n\ntest')
+    expect(entToStr(output.text, output.entities?.[0])).toEqual('test\n\ntest')
+    expect(entToStr(output.text, output.entities?.[1])).toEqual('test')
+    expect(entToStr(output.text, output.entities?.[2])).toEqual('test')
+    expect(output.entities?.[3]).toEqual(undefined)
+  })
+})
+
+function entToStr(str: string, ent?: Entity) {
+  if (!ent) {
+    return ''
+  }
+  return str.slice(ent.index.start, ent.index.end)
+}
@@ -0,0 +1,123 @@
+import {RichText} from '../../../src/lib/strings/rich-text'
+
+describe('richText.insert', () => {
+  const input = new RichText('hello world', [
+    {index: {start: 2, end: 7}, type: '', value: ''},
+  ])
+
+  it('correctly adjusts entities (scenario A - before)', () => {
+    const output = input.clone().insert(0, 'test')
+    expect(output.text).toEqual('testhello world')
+    expect(output.entities?.[0].index.start).toEqual(6)
+    expect(output.entities?.[0].index.end).toEqual(11)
+    expect(
+      output.text.slice(
+        output.entities?.[0].index.start,
+        output.entities?.[0].index.end,
+      ),
+    ).toEqual('llo w')
+  })
+
+  it('correctly adjusts entities (scenario B - inner)', () => {
+    const output = input.clone().insert(4, 'test')
+    expect(output.text).toEqual('helltesto world')
+    expect(output.entities?.[0].index.start).toEqual(2)
+    expect(output.entities?.[0].index.end).toEqual(11)
+    expect(
+      output.text.slice(
+        output.entities?.[0].index.start,
+        output.entities?.[0].index.end,
+      ),
+    ).toEqual('lltesto w')
+  })
+
+  it('correctly adjusts entities (scenario C - after)', () => {
+    const output = input.clone().insert(8, 'test')
+    expect(output.text).toEqual('hello wotestrld')
+    expect(output.entities?.[0].index.start).toEqual(2)
+    expect(output.entities?.[0].index.end).toEqual(7)
+    expect(
+      output.text.slice(
+        output.entities?.[0].index.start,
+        output.entities?.[0].index.end,
+      ),
+    ).toEqual('llo w')
+  })
+})
+
+describe('richText.delete', () => {
+  const input = new RichText('hello world', [
+    {index: {start: 2, end: 7}, type: '', value: ''},
+  ])
+
+  it('correctly adjusts entities (scenario A - entirely outer)', () => {
+    const output = input.clone().delete(0, 9)
+    expect(output.text).toEqual('ld')
+    expect(output.entities?.length).toEqual(0)
+  })
+
+  it('correctly adjusts entities (scenario B - entirely after)', () => {
+    const output = input.clone().delete(7, 11)
+    expect(output.text).toEqual('hello w')
+    expect(output.entities?.[0].index.start).toEqual(2)
+    expect(output.entities?.[0].index.end).toEqual(7)
+    expect(
+      output.text.slice(
+        output.entities?.[0].index.start,
+        output.entities?.[0].index.end,
+      ),
+    ).toEqual('llo w')
+  })
+
+  it('correctly adjusts entities (scenario C - partially after)', () => {
+    const output = input.clone().delete(4, 11)
+    expect(output.text).toEqual('hell')
+    expect(output.entities?.[0].index.start).toEqual(2)
+    expect(output.entities?.[0].index.end).toEqual(4)
+    expect(
+      output.text.slice(
+        output.entities?.[0].index.start,
+        output.entities?.[0].index.end,
+      ),
+    ).toEqual('ll')
+  })
+
+  it('correctly adjusts entities (scenario D - entirely inner)', () => {
+    const output = input.clone().delete(3, 5)
+    expect(output.text).toEqual('hel world')
+    expect(output.entities?.[0].index.start).toEqual(2)
+    expect(output.entities?.[0].index.end).toEqual(5)
+    expect(
+      output.text.slice(
+        output.entities?.[0].index.start,
+        output.entities?.[0].index.end,
+      ),
+    ).toEqual('l w')
+  })
+
+  it('correctly adjusts entities (scenario E - partially before)', () => {
+    const output = input.clone().delete(1, 5)
+    expect(output.text).toEqual('h world')
+    expect(output.entities?.[0].index.start).toEqual(1)
+    expect(output.entities?.[0].index.end).toEqual(3)
+    expect(
+      output.text.slice(
+        output.entities?.[0].index.start,
+        output.entities?.[0].index.end,
+      ),
+    ).toEqual(' w')
+  })
+
+  it('correctly adjusts entities (scenario F - entirely before)', () => {
+    const output = input.clone().delete(0, 2)
+    expect(output.text).toEqual('llo world')
+    expect(output.entities?.[0].index.start).toEqual(0)
+    expect(output.entities?.[0].index.end).toEqual(5)
+    expect(
+      output.text.slice(
+        output.entities?.[0].index.start,
+        output.entities?.[0].index.end,
+      ),
+    ).toEqual('llo w')
+  })
+})