PhosCity · September 20, 2024 05:15 · PhosCity · Sep 20, 2024
diff --git a/l0.ASSWipe.moon b/l0.ASSWipe.moon
 export script_name = "ASSWipe"
 export script_description = "Performs script cleanup, removes unnecessary tags and lines."
 export script_version = "0.5.0"
 export script_author = "line0"
 export script_namespace = "l0.ASSWipe"

 DependencyControl = require "l0.DependencyControl"
 version = DependencyControl{
  feed: "https://raw.githubusercontent.com/TypesettingTools/line0-Aegisub-Scripts/master/DependencyControl.json",
  {
    {"a-mo.LineCollection", version: "1.3.0", url: "https://github.com/TypesettingTools/Aegisub-Motion",
      feed: "https://raw.githubusercontent.com/TypesettingTools/Aegisub-Motion/DepCtrl/DependencyControl.json"},
    {"a-mo.ConfigHandler", version: "1.1.4", url: "https://github.com/TypesettingTools/Aegisub-Motion",
      feed: "https://raw.githubusercontent.com/TypesettingTools/Aegisub-Motion/DepCtrl/DependencyControl.json"},
    {"l0.ASSFoundation", version: "0.5.0", url: "https://github.com/TypesettingTools/ASSFoundation",
      feed: "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json"},
    {"l0.Functional", version: "0.5.0", url: "https://github.com/TypesettingTools/Functional",
      feed: "https://raw.githubusercontent.com/TypesettingTools/Functional/master/DependencyControl.json"},
    {"SubInspector.Inspector", version: "0.7.2", url: "https://github.com/TypesettingTools/SubInspector",
      feed: "https://raw.githubusercontent.com/TypesettingTools/SubInspector/master/DependencyControl.json"},
    "json"
  }
 }
 LineCollection, ConfigHandler, ASS, Functional, SubInspector, json = version\requireModules!
 ffms, min, concat, sort = aegisub.frame_from_ms, math.min, table.concat, table.sort
 {:list, :math, :string, :table, :unicode, :util, :re } = Functional
 logger = version\getLogger!

 reportMsg = [[
 Done. Processed %d lines in %d seconds.
 — Cleaned %d lines (%d%%)
 — Removed %d invisible lines (%d%%)
 — Combined %d consecutive identical lines (%d%%)
 — Filtered %d clips and %d occurences of junk data
 — Purged %d invisible contours (%d in drawings, %d in clips)
 — Failed to purge %d invisible contours due to rendering inconsistencies
 — Converted %d drawings/clips to floating-point
 — Filtered %d records of extra data
 — Total space saved: %.2f KB
 ]]

 hints = {
  cleanLevel: [[
 0: no cleaning
 1: remove empty tag sections
 2: deduplicate tags inside sections
 3: deduplicate tags globally,
 4: remove tags matching the style defaults and otherwise ineffective tags]]
  tagsToKeep: "Don't remove these tags even if they match the style defaults for the line."
  filterClips: "Removes clips that don't affect the rendered output."
  removeInvisible: "Deletes lines that don't generate any visible output."
  combineLines: "Merges non-animated lines that render to an identical result and have consecutive times (without overlaps or gaps)."
  removeJunk: "Removes any 'in-line comments' and things not starting with a \\ from tag sections."
  removeUnknown: "Removes any unknown tags from the line"
  scale2float: "Converts drawings and clips with a scale parameter to a floating-point representation."
  tagSortOrder: "Determines the order cleaned tags will be ordered inside a tag section. Resets always go first, transforms last."
  fixDrawings: "Removes extraneous ordinates from broken drawings to make them parseable. May or may not changed the rendered output."
  purgeContoursDraw: "Removes all contours of a drawing that are not visible on the canvas."
  purgeContoursClip: "Removes all contours of a clip that do not affect the appearance of the line."
  stripComments: "Removes any comments encapsulated in {curly brackets}."
  purgeContoursIgnoreHashMismatch: "Removes invisible contours even if it causes a SubInspector hash mismatch when comparing the result to the original line. This may or may not visually affect your drawing, so never use this option unsupervised!"
 }

 defaultSortOrder = [[
 \an, \pos, \move, \org, \fscx, \fscy, \frz, \fry, \frx, \fax, \fay, \fn, \fs, \fsp, \b, \i, \u, \s, \bord, \xbord, \ybord,
 \shad, \xshad, \yshad, \1c, \2c, \3c, \4c, \alpha, \1a, \2a, \3a, \4a, \blur, \be, \fad, \fade, clip_rect, iclip_rect,
 clip_vect, iclip_vect, \q, \p, \k, \kf, \K, \ko, junk, unknown
 ]]
 karaokeTags = table.concat table.pluck(table.filter(ASS.tagMap, (tag) -> tag.props.karaoke), "overrideName"), ", "

 -- to be moved into ASSFoundation.Functional
 sortWithKeys = (tbl, comparator) ->
  -- shellsort written by Rici Lake
  -- c/p from http://lua-users.org/wiki/LuaSorting, with index argument added to comparator
  incs = { 1391376, 463792, 198768, 86961, 33936, 13776, 4592, 1968, 861, 336, 112, 48, 21, 7, 3, 1 }
  n = #tbl
  for h in *incs
    for i = h+1, n
      a = tbl[i]
      for j = i-h, 1, -h
        b = tbl[j]
        break unless comparator a, b, i, j
        tbl[i] = b
        i = j
      tbl[i] = a
  return tbl


 -- returns if we can merge line b into a while still maintain b's layer order
 isMergeable = (a, b, linesByFrame) ->
  for i = b.firstFrame, b.lastFrame
    group = linesByFrame[i]
    local pos

    unless group.sorted
      -- ensure line group is sorted by layer blending order
      sortWithKeys group, (x, y, i, j) ->
        pos or= i if x == b
        return x.layer < y.layer or x.layer == y.layer and i < j

      group.sorted = true

    -- get line position in blending order
    pos or= i for i, v in ipairs group when v == b

    -- as b is merged into a, it gets a's layer number
    -- so we can only merge if the new layer number does not change the blending order in any of the frames b is visible in
    lower = group[pos-1]
    return false unless (not lower or a.layer > lower.layer or a.layer == lower.layer and a.number > lower.number)

    higher = group[pos+1]
    return false unless (not higher or a.layer < higher.layer or a.layer == higher.layer and a.number < higher.number)

  return true

 mergeLines = (lines, start, cmbCnt, bytes, linesByFrame) ->
  -- queue merged lines for deletion and collect statistics
  if lines[start].merged
    return lines[start], cmbCnt+1, bytes + #lines[start].raw + 1

  -- merge applicable lines into first mergeable by extending its end time
  -- then mark all merged lines
  merged = lines[start]
  for i=start+1,lines.n
    line = lines[i]
    break if line.merged or line.start_time != merged.end_time or not isMergeable merged, line, linesByFrame
    lines[i].merged = true
    merged.end_time = lines[i].end_time

    -- update lines by frame index
    for f = line.firstFrame, line.lastFrame
      group = linesByFrame[f]
      pos = i for i, v in ipairs group when v == line
      group[pos] = merged

  return nil, cmbCnt, bytes


 removeInvisibleContoursOptCollectBounds = (contour, _, sectionContourIndex, sliceContourIndex, _, sliceRawContours, sliceSize, linePre, linePost, isAnimated, boundsBatch) ->
  prevContours = concat sliceRawContours, " ", 1, sliceContourIndex-1
  nextContours = sliceContourIndex <= sliceSize and concat(sliceRawContours, " ", sliceContourIndex+1) or ""
  text = "#{linePre}#{prevContours}#{(#prevContours == 0 or #nextContours == 0) and "" or " "}#{nextContours}#{linePost}"
  boundsBatch\add contour.parent.parent, text, sectionContourIndex, isAnimated
  rawContour = sliceRawContours[sliceContourIndex]

 removeInvisibleContoursOptPurge = (contour, contours, i, _, _, allBounds, orgBounds) ->
  bounds, allBounds[i] = allBounds[i]
  return false if orgBounds\equal bounds

 removeInvisibleContoursOpt = (section, orgBounds) ->
  cutOff, ass, contourCnt = false, section.parent, #section.contours

  sliceSize = math.ceil 10e4 / contourCnt
  if contourCnt > 100
    logger\hint "Cleaning complex drawing with %d contours (slice size: %s)...", contourCnt, sliceSize

  selectSurroundingSections = (sect, _, _, _, toTheLeft) ->
    if toTheLeft
      cutOff = true if section == sect
    else
      if section == sect
        cutOff = false
        return false

    return not cutOff


  lineStringPre, drwState = ass\getString nil, nil, selectSurroundingSections, false, true
  lineStringPost = ass\getString nil, drwState, selectSurroundingSections, false, false

  allBounds, sliceContours, sliceStartIndex = {}
  isAnimated = ass\isAnimated!

  for sliceStartIndex = 1, contourCnt, sliceSize
    sliceEndIndex = min sliceStartIndex+sliceSize-1, contourCnt
    sliceContours = [cnt\getTagParams! for cnt in *section.contours[sliceStartIndex, sliceEndIndex]]
    boundsBatch = ASS.LineBoundsBatch!

    section\callback removeInvisibleContoursOptCollectBounds, sliceStartIndex, sliceEndIndex, nil, nil,
      sliceContours, sliceSize, lineStringPre, lineStringPost, isAnimated, boundsBatch

    boundsBatch\run true, allBounds
    lineStringPre ..= concat sliceContours, " "
    boundsBatch = nil
    collectgarbage!

  _, purgeCnt = section\callback removeInvisibleContoursOptPurge, nil, nil, nil, nil, allBounds, orgBounds
  return purgeCnt

 stripComments = () -> false

 process = (sub, sel, res) ->
  ASS.config.fixDrawings = res.fixDrawings
  lines = LineCollection sub, sel
  linesToDelete, delCnt, linesToCombine, cmbCnt, lineCnt, debugError = {}, 0, {}, 0, #lines.lines, false
  tagNames = res.filterClips and util.copy(ASS.tagNames.clips) or {}
  tagNames[#tagNames+1] = res.removeJunk and "junk"
  tagNames[#tagNames+1] = res.removeUnknown and "unknown"
  stats = { bytes: 0, junk: 0, clips: 0, start: os.time!, cleaned: 0,
        scale2float: 0, contoursDraw: 0, contoursDrawSkipped: 0, contoursClip: 0, extra: 0 }
  linesByFrame = {}

  -- create proper tag name lists from user input which may be override tag names or mixed
  res.tagsToKeep = ASS\getTagNames string.split res.tagsToKeep, ",%s", nil, false
  res.tagSortOrder = ASS\getTagNames string.split res.tagSortOrder, ",%s", nil, false
  res.mergeConsecutiveExcept = ASS\getTagNames string.split res.mergeConsecutiveExcept, ",%s", nil, false
  res.extraDataFilter = string.split res.extraDataFilter, ",%s", nil, false

  callback = (lines, line, i) ->
    aegisub.cancel! if aegisub.progress.is_cancelled!
    aegisub.progress.task "Cleaning %d of %d lines..."\format i, lineCnt if i%10==0
    aegisub.progress.set 100*i/lineCnt

    unless line.styleRef
      logger\warn "WARNING: Line #%d is using undefined style '%s', skipping...\n— %s", i, line.style, line.text
      return

    -- filter extra data
    if line.extra and res.extraDataMode != "Keep all"
      removed, r = switch res.extraDataMode
        when "Remove all"
          removed = line.extra
          line.extra = nil
          removed, table.length removed
        when "Remove all except"
          table.removeKeysExcept line.extra, res.extraDataFilter
        when "Keep all except"
          table.removeKeys line.extra, res.extraDataFilter
      stats.extra += r

    success, data = pcall ASS\parse, line
    unless success
      logger\warn "Couldn't parse line #%d: %s", i, data
      return

    -- it is essential to run SubInspector on a ASSFoundation-built line (rather than the original)
    -- because ASSFoundation rounds tag values to a sane precision, which is not visible but
    -- will produce a hash mismatch compared to the original line. However we must avoid that to
    -- not trigger the ASSWipe bug detection
    orgText, oldBounds = line.text, data\getLineBounds false, true
    orgTextParsed = oldBounds.rawText
    orgTagTypes = ["#{tag.__tag.name}(#{table.concat({tag\getTagParams!}, ",")})" for tag in *data\getTags!]


    removeInvisibleContours = (contour) ->
      contour.disabled = true
      if oldBounds\equal data\getLineBounds!
        if contour.parent.class == ASS.Section.Drawing
          stats.contoursDraw += 1
        else stats.contoursClip += 1
        return false
      contour.disabled = false


    -- remove invisible lines
    if res.removeInvisible and oldBounds.w == 0
      stats.bytes += #line.raw + 1
      delCnt += 1
      linesToDelete[delCnt], line.ASS = line
      return


    purgedContourCount = 0
    if res.purgeContoursDraw or res.scale2float
      cb = (section) ->
        -- remove invisible contours from drawings
        if res.purgeContoursDraw
          purgedContourCount = removeInvisibleContoursOpt section, oldBounds
        -- un-scale drawings
        if res.scale2float and section.scale > 1
          section.scale\set 1
          stats.scale2float += 1

      data\callback cb, ASS.Section.Drawing
      if purgedContourCount > 0
        if res.purgeContoursDraw and not res.purgeContoursIgnoreHashMismatch and not data\getLineBounds!\equal oldBounds
          line.text = orgText
          data = ASS\parse line
          stats.contoursDrawSkipped += purgedContourCount
        else
          stats.contoursDraw += purgedContourCount
          oldBounds = data\getLineBounds! if res.purgeContoursIgnoreHashMismatch

    -- pogressively build a table of visible lines by frame
    -- which is required to check mergeability of consecutive identical lines
    if res.combineLines
      line.firstFrame = ffms line.start_time
      line.lastFrame = -1 + ffms line.end_time
      for i = line.firstFrame, line.lastFrame
        lbf = linesByFrame[i]
        if lbf
          lbf[lbf.n+1] = line
          lbf.n += 1
        else linesByFrame[i] = {line, n: 1}

      -- collect lines to combine
      if oldBounds.firstHash and not oldBounds.animated
        ltc = linesToCombine[oldBounds.firstHash]
        if ltc
          ltc[ltc.n+1] = line
          ltc.n += 1
        else linesToCombine[oldBounds.firstHash] = {line, n: 1}

    mergeConsecutiveTagSections = if not res.mergeConsecutive
      false
    elseif #res.mergeConsecutiveExcept == 0
      true
    else
      exceptions = list.makeSet res.mergeConsecutiveExcept
      (sourceSection, targetSection) ->
        predicate = (tag) -> exceptions[tag.__tag.name]
        not table.find(sourceSection.tags, predicate) and not table.find targetSection.tags, predicate
    -- clean tags
    data\cleanTags res.cleanLevel, mergeConsecutiveTagSections, res.tagsToKeep, res.tagSortOrder
    newBounds = data\getLineBounds!

    if res.stripComments
      data\stripComments!
      --data\callback stripComments, ASS.Section.Comment

    if res.filterClips or res.removeJunk
      data\modTags tagNames, (tag) ->
        -- remove junk
        if tag.class == ASS.Tag.Unknown
          stats.junk += 1
          return false

        if tag.class == ASS.Tag.ClipVect
          -- un-scale clips
          if res.scale2float and tag.scale>1
            tag.scale\set 1
            stats.scale2float += 1
          -- purge ineffective contours from clips
          if res.purgeContoursClip
            tag\callback removeInvisibleContours

        -- filter clips
        tag.disabled = true
        if data\getLineBounds!\equal newBounds
          stats.clips += 1
          return false
        tag.disabled = false

    data\commit nil, res.cleanLevel == 0
    -- reclaim some memory
    line.ASS, line.undoText = nil

    if orgText != line.text
      if not newBounds\equal oldBounds
        debugError = true
        logger\warn "Cleaning affected output on line #%d, rolling back...", line.humanizedNumber
        logger\warn "—— Before: %s\n—— Parsed: %s\n—— After: %s\n—— Style: %s\n—— Tags: %s", orgText, orgTextParsed, line.text, line.styleRef.name, table.concat orgTagTypes, "; "
        logger\warn "—— Hash Before: %s (%s); Hash After: %s (%s)\n",
              oldBounds.firstHash, oldBounds.animated and "animated" or "static",
              newBounds.firstHash, newBounds.animated and "animated" or "static"
        line.text = orgText
      elseif #line.text < #orgText
        stats.cleaned += 1
        stats.bytes += #orgText - #line.text

    aegisub.cancel! if aegisub.progress.is_cancelled!
  lines\runCallback callback, true

  -- sort lines which are to be combined by time
  sortFunc = (a, b) ->
    return true if a.start_time < b.start_time
    return false if a.start_time > b.start_time
    return true if a.layer < b.layer
    return false if a.layer > b.layer
    return true if a.number < b.number
    return false

  linesToCombineSorted, l = {}, 1
  for _, group in pairs linesToCombine
    continue if group.n < 2
    sort group, sortFunc
    linesToCombineSorted[l] = group
    l += 1
  sort linesToCombineSorted, (a, b) -> sortFunc a[1], b[1]

  -- combine lines
  for group in *linesToCombineSorted
    for j=1, group.n
      linesToDelete[delCnt+cmbCnt+1], cmbCnt, stats.bytes = mergeLines group, j, cmbCnt, stats.bytes, linesByFrame

  lines\replaceLines!
  lines\deleteLines linesToDelete

  logger\warn json.encode {Styles: lines.styles, Configuration: res} if debugError
  logger\warn reportMsg, lineCnt, os.time!-stats.start, stats.cleaned, 100*stats.cleaned/lineCnt,
    delCnt, 100*delCnt/lineCnt, cmbCnt, 100*cmbCnt/lineCnt, stats.clips, stats.junk,
    stats.contoursClip+stats.contoursDraw, stats.contoursDraw, stats.contoursClip, stats.contoursDrawSkipped,
    stats.scale2float, stats.extra, stats.bytes/1000

  if debugError
    logger\warn [[However, ASSWipe possibly encountered bugs while cleaning.
          Affected lines have been rolled back to their previous state, so your script is most likely fine.
          Please copy the whole log window contents and send them to line0.]]


  return lines\getSelection!


 showDialog = (sub, sel, res) ->
  dlg = {
    main: {
      removeInvisible:    class: "checkbox", x: 0, y: 0, width: 2,  height: 1, value: true, config: true, label: "Remove invisible lines", hint: hints.removeInvisible
      combineLines:       class: "checkbox", x: 0, y: 1, width: 2,  height: 1, value: true, config: true, label: "Combine consecutive identical lines", hint: hints.combineLines
      mergeConsecutive:   class: "checkbox", x: 2, y: 0, width: 12,  height: 1, value: true, config: true, label: "Merge consecutive tag sections unless it contains any of:", hint: hints.mergeConsecutive
      mergeConsecutiveExcept: class: "textbox", x: 2, y: 1, width: 12, height: 2, value: karaokeTags, config: true, hint: hints.mergeConsecutiveExcept

      cleanLevelLabel:    class: "label",    x: 0, y: 4, width: 1,  height: 1, label: "Tag cleanup level: "
      cleanLevel:         class: "intedit",  x: 1, y: 4, width: 1,  height: 1, min: 0, max: 4, value: 4, config: true, hint: hints.cleanLevel
      tagsToKeepLabel:    class: "label",    x: 4, y: 4, width: 1,  height: 1, label: "Keep default tags: "
      tagsToKeep:         class: "textbox",  x: 4, y: 5, width: 10, height: 2, value: "\\pos", config:true, hint: hints.tagsToKeep
      tagSortOrderLabel:  class: "label",    x: 4, y: 7, width: 1,  height: 1, label: "Tag sort order: "
      stripComments:      class: "checkbox", x: 0, y: 5, width: 2,  height: 1, value: true, config: true, label: "Strip comments", hint: hints.stripComments
      removeJunk:         class: "checkbox", x: 0, y: 6, width: 2,  height: 1, value: true, config: true, label: "Remove junk from tag sections", hint: hints.removeJunk
      removeUnknown:      class: "checkbox", x: 0, y: 7, width: 2,  height: 1, value: true, config: true, label: "Remove unknown tags from tag sections", hint: hints.removeJunk
      tagSortOrder:       class: "textbox",  x: 4, y: 8, width: 10, height: 3, value: defaultSortOrder, config: true, hint: hints.tagSortOrder

      filterClips:        class: "checkbox", x: 0, y: 11, width: 2,  height: 1, value: true, config: true, label: "Filter clips", hint: hints.filterClips
      scale2float:        class: "checkbox", x: 0, y: 12, width: 2,  height: 1, value: true, config: true, label: "Un-scale drawings and clips", hint: hints.scale2float
      fixDrawings:        class: "checkbox", x: 0, y: 13, width: 2,  height: 1, value: false, config: true, label: "Try to fix broken drawings", hint: hints.fixDrawings
      purgeContoursLabel: class: "label",    x: 0, y: 14, width: 2,  height: 1, label: "Purge invisible contours: "
      purgeContoursDraw:  class: "checkbox", x: 4, y: 14, width: 3,  height: 1, value: false, config: true, label: "from drawings", hint: hints.purgeContoursDraw
      purgeContoursClip:  class: "checkbox", x: 7, y: 14, width: 6,  height: 1, value: false, config: true, label: "from clips", hint: hints.purgeContoursClip
      purgeContoursIgnoreHashMismatch: class: "checkbox", x: 4, y: 15, width: 9, height: 1, value: false, config: true, label: "ignore rendering inconsistencies", hint: hints.purgeContoursIgnoreHashMismatch
      extraDataLabel:     class: "label",    x: 0, y: 17, width: 1,  height: 1, label: "Filter extra data: "
      extraDataMode:      class: "dropdown", x: 1, y: 17, width: 1,  height: 1, value: "Keep All", config: true, items: {"Keep all", "Remove all", "Keep all except", "Remove all except"}, hint: hints.extraData
      extraDataFilter:    class: "textbox",  x: 4, y: 17, width: 10, height: 3, value: "", config: true, hint: hints.extraData
      quirksModeLabel:    class: "label",    x: 0, y: 21, width: 2,  height: 1, label: "Quirks mode: "
      quirksMode:         class: "dropdown", x: 4, y: 21, width: 2,  height: 1, value: "VSFilter", config: true, items: [k for k, v in pairs ASS.Quirks], hint: hints.quirksMode
    }
  }
  options = ConfigHandler dlg, version.configFile, false, script_version, version.configDir
  options\read!
  options\updateInterface "main"
  btn, res = aegisub.dialog.display dlg.main
  if btn
    options\updateConfiguration res, "main"
    options\write!
    ASS.config.quirks = ASS.Quirks[res.quirksMode]
    process sub, sel, res

 version\registerMacro showDialog, ->
  if aegisub.project_properties!.video_file == ""
    return false, "A video must be loaded to run #{script_name}."
  else return true, script_description
	export script_name = "ASSWipe"
	export script_description = "Performs script cleanup, removes unnecessary tags and lines."
	export script_version = "0.5.0"
	export script_author = "line0"
	export script_namespace = "l0.ASSWipe"

	DependencyControl = require "l0.DependencyControl"
	version = DependencyControl{
	feed: "https://raw.githubusercontent.com/TypesettingTools/line0-Aegisub-Scripts/master/DependencyControl.json",
	{
	{"a-mo.LineCollection", version: "1.3.0", url: "https://github.com/TypesettingTools/Aegisub-Motion",
	feed: "https://raw.githubusercontent.com/TypesettingTools/Aegisub-Motion/DepCtrl/DependencyControl.json"},
	{"a-mo.ConfigHandler", version: "1.1.4", url: "https://github.com/TypesettingTools/Aegisub-Motion",
	feed: "https://raw.githubusercontent.com/TypesettingTools/Aegisub-Motion/DepCtrl/DependencyControl.json"},
	{"l0.ASSFoundation", version: "0.5.0", url: "https://github.com/TypesettingTools/ASSFoundation",
	feed: "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json"},
	{"l0.Functional", version: "0.5.0", url: "https://github.com/TypesettingTools/Functional",
	feed: "https://raw.githubusercontent.com/TypesettingTools/Functional/master/DependencyControl.json"},
	{"SubInspector.Inspector", version: "0.7.2", url: "https://github.com/TypesettingTools/SubInspector",
	feed: "https://raw.githubusercontent.com/TypesettingTools/SubInspector/master/DependencyControl.json"},
	"json"
	}
	}
	LineCollection, ConfigHandler, ASS, Functional, SubInspector, json = version\requireModules!
	ffms, min, concat, sort = aegisub.frame_from_ms, math.min, table.concat, table.sort
	{:list, :math, :string, :table, :unicode, :util, :re } = Functional
	logger = version\getLogger!

	reportMsg = [[
	Done. Processed %d lines in %d seconds.
	— Cleaned %d lines (%d%%)
	— Removed %d invisible lines (%d%%)
	— Combined %d consecutive identical lines (%d%%)
	— Filtered %d clips and %d occurences of junk data
	— Purged %d invisible contours (%d in drawings, %d in clips)
	— Failed to purge %d invisible contours due to rendering inconsistencies
	— Converted %d drawings/clips to floating-point
	— Filtered %d records of extra data
	— Total space saved: %.2f KB
	]]

	hints = {
	cleanLevel: [[
	0: no cleaning
	1: remove empty tag sections
	2: deduplicate tags inside sections
	3: deduplicate tags globally,
	4: remove tags matching the style defaults and otherwise ineffective tags]]
	tagsToKeep: "Don't remove these tags even if they match the style defaults for the line."
	filterClips: "Removes clips that don't affect the rendered output."
	removeInvisible: "Deletes lines that don't generate any visible output."
	combineLines: "Merges non-animated lines that render to an identical result and have consecutive times (without overlaps or gaps)."
	removeJunk: "Removes any 'in-line comments' and things not starting with a \\ from tag sections."
	removeUnknown: "Removes any unknown tags from the line"
	scale2float: "Converts drawings and clips with a scale parameter to a floating-point representation."
	tagSortOrder: "Determines the order cleaned tags will be ordered inside a tag section. Resets always go first, transforms last."
	fixDrawings: "Removes extraneous ordinates from broken drawings to make them parseable. May or may not changed the rendered output."
	purgeContoursDraw: "Removes all contours of a drawing that are not visible on the canvas."
	purgeContoursClip: "Removes all contours of a clip that do not affect the appearance of the line."
	stripComments: "Removes any comments encapsulated in {curly brackets}."
	purgeContoursIgnoreHashMismatch: "Removes invisible contours even if it causes a SubInspector hash mismatch when comparing the result to the original line. This may or may not visually affect your drawing, so never use this option unsupervised!"
	}

	defaultSortOrder = [[
	\an, \pos, \move, \org, \fscx, \fscy, \frz, \fry, \frx, \fax, \fay, \fn, \fs, \fsp, \b, \i, \u, \s, \bord, \xbord, \ybord,
	\shad, \xshad, \yshad, \1c, \2c, \3c, \4c, \alpha, \1a, \2a, \3a, \4a, \blur, \be, \fad, \fade, clip_rect, iclip_rect,
	clip_vect, iclip_vect, \q, \p, \k, \kf, \K, \ko, junk, unknown
	]]
	karaokeTags = table.concat table.pluck(table.filter(ASS.tagMap, (tag) -> tag.props.karaoke), "overrideName"), ", "

	-- to be moved into ASSFoundation.Functional
	sortWithKeys = (tbl, comparator) ->
	-- shellsort written by Rici Lake
	-- c/p from http://lua-users.org/wiki/LuaSorting, with index argument added to comparator
	incs = { 1391376, 463792, 198768, 86961, 33936, 13776, 4592, 1968, 861, 336, 112, 48, 21, 7, 3, 1 }
	n = #tbl
	for h in *incs
	for i = h+1, n
	a = tbl[i]
	for j = i-h, 1, -h
	b = tbl[j]
	break unless comparator a, b, i, j
	tbl[i] = b
	i = j
	tbl[i] = a
	return tbl


	-- returns if we can merge line b into a while still maintain b's layer order
	isMergeable = (a, b, linesByFrame) ->
	for i = b.firstFrame, b.lastFrame
	group = linesByFrame[i]
	local pos

	unless group.sorted
	-- ensure line group is sorted by layer blending order
	sortWithKeys group, (x, y, i, j) ->
	pos or= i if x == b
	return x.layer < y.layer or x.layer == y.layer and i < j

	group.sorted = true

	-- get line position in blending order
	pos or= i for i, v in ipairs group when v == b

	-- as b is merged into a, it gets a's layer number
	-- so we can only merge if the new layer number does not change the blending order in any of the frames b is visible in
	lower = group[pos-1]
	return false unless (not lower or a.layer > lower.layer or a.layer == lower.layer and a.number > lower.number)

	higher = group[pos+1]
	return false unless (not higher or a.layer < higher.layer or a.layer == higher.layer and a.number < higher.number)

	return true

	mergeLines = (lines, start, cmbCnt, bytes, linesByFrame) ->
	-- queue merged lines for deletion and collect statistics
	if lines[start].merged
	return lines[start], cmbCnt+1, bytes + #lines[start].raw + 1

	-- merge applicable lines into first mergeable by extending its end time
	-- then mark all merged lines
	merged = lines[start]
	for i=start+1,lines.n
	line = lines[i]
	break if line.merged or line.start_time != merged.end_time or not isMergeable merged, line, linesByFrame
	lines[i].merged = true
	merged.end_time = lines[i].end_time

	-- update lines by frame index
	for f = line.firstFrame, line.lastFrame
	group = linesByFrame[f]
	pos = i for i, v in ipairs group when v == line
	group[pos] = merged

	return nil, cmbCnt, bytes


	removeInvisibleContoursOptCollectBounds = (contour, _, sectionContourIndex, sliceContourIndex, _, sliceRawContours, sliceSize, linePre, linePost, isAnimated, boundsBatch) ->
	prevContours = concat sliceRawContours, " ", 1, sliceContourIndex-1
	nextContours = sliceContourIndex <= sliceSize and concat(sliceRawContours, " ", sliceContourIndex+1) or ""
	text = "#{linePre}#{prevContours}#{(#prevContours == 0 or #nextContours == 0) and "" or " "}#{nextContours}#{linePost}"
	boundsBatch\add contour.parent.parent, text, sectionContourIndex, isAnimated
	rawContour = sliceRawContours[sliceContourIndex]

	removeInvisibleContoursOptPurge = (contour, contours, i, _, _, allBounds, orgBounds) ->
	bounds, allBounds[i] = allBounds[i]
	return false if orgBounds\equal bounds

	removeInvisibleContoursOpt = (section, orgBounds) ->
	cutOff, ass, contourCnt = false, section.parent, #section.contours

	sliceSize = math.ceil 10e4 / contourCnt
	if contourCnt > 100
	logger\hint "Cleaning complex drawing with %d contours (slice size: %s)...", contourCnt, sliceSize

	selectSurroundingSections = (sect, _, _, _, toTheLeft) ->
	if toTheLeft
	cutOff = true if section == sect
	else
	if section == sect
	cutOff = false
	return false

	return not cutOff


	lineStringPre, drwState = ass\getString nil, nil, selectSurroundingSections, false, true
	lineStringPost = ass\getString nil, drwState, selectSurroundingSections, false, false

	allBounds, sliceContours, sliceStartIndex = {}
	isAnimated = ass\isAnimated!

	for sliceStartIndex = 1, contourCnt, sliceSize
	sliceEndIndex = min sliceStartIndex+sliceSize-1, contourCnt
	sliceContours = [cnt\getTagParams! for cnt in *section.contours[sliceStartIndex, sliceEndIndex]]
	boundsBatch = ASS.LineBoundsBatch!

	section\callback removeInvisibleContoursOptCollectBounds, sliceStartIndex, sliceEndIndex, nil, nil,
	sliceContours, sliceSize, lineStringPre, lineStringPost, isAnimated, boundsBatch

	boundsBatch\run true, allBounds
	lineStringPre ..= concat sliceContours, " "
	boundsBatch = nil
	collectgarbage!

	_, purgeCnt = section\callback removeInvisibleContoursOptPurge, nil, nil, nil, nil, allBounds, orgBounds
	return purgeCnt

	stripComments = () -> false

	process = (sub, sel, res) ->
	ASS.config.fixDrawings = res.fixDrawings
	lines = LineCollection sub, sel
	linesToDelete, delCnt, linesToCombine, cmbCnt, lineCnt, debugError = {}, 0, {}, 0, #lines.lines, false
	tagNames = res.filterClips and util.copy(ASS.tagNames.clips) or {}
	tagNames[#tagNames+1] = res.removeJunk and "junk"
	tagNames[#tagNames+1] = res.removeUnknown and "unknown"
	stats = { bytes: 0, junk: 0, clips: 0, start: os.time!, cleaned: 0,
	scale2float: 0, contoursDraw: 0, contoursDrawSkipped: 0, contoursClip: 0, extra: 0 }
	linesByFrame = {}

	-- create proper tag name lists from user input which may be override tag names or mixed
	res.tagsToKeep = ASS\getTagNames string.split res.tagsToKeep, ",%s", nil, false
	res.tagSortOrder = ASS\getTagNames string.split res.tagSortOrder, ",%s", nil, false
	res.mergeConsecutiveExcept = ASS\getTagNames string.split res.mergeConsecutiveExcept, ",%s", nil, false
	res.extraDataFilter = string.split res.extraDataFilter, ",%s", nil, false

	callback = (lines, line, i) ->
	aegisub.cancel! if aegisub.progress.is_cancelled!
	aegisub.progress.task "Cleaning %d of %d lines..."\format i, lineCnt if i%10==0
	aegisub.progress.set 100*i/lineCnt

	unless line.styleRef
	logger\warn "WARNING: Line #%d is using undefined style '%s', skipping...\n— %s", i, line.style, line.text
	return

	-- filter extra data
	if line.extra and res.extraDataMode != "Keep all"
	removed, r = switch res.extraDataMode
	when "Remove all"
	removed = line.extra
	line.extra = nil
	removed, table.length removed
	when "Remove all except"
	table.removeKeysExcept line.extra, res.extraDataFilter
	when "Keep all except"
	table.removeKeys line.extra, res.extraDataFilter
	stats.extra += r

	success, data = pcall ASS\parse, line
	unless success
	logger\warn "Couldn't parse line #%d: %s", i, data
	return

	-- it is essential to run SubInspector on a ASSFoundation-built line (rather than the original)
	-- because ASSFoundation rounds tag values to a sane precision, which is not visible but
	-- will produce a hash mismatch compared to the original line. However we must avoid that to
	-- not trigger the ASSWipe bug detection
	orgText, oldBounds = line.text, data\getLineBounds false, true
	orgTextParsed = oldBounds.rawText
	orgTagTypes = ["#{tag.__tag.name}(#{table.concat({tag\getTagParams!}, ",")})" for tag in *data\getTags!]


	removeInvisibleContours = (contour) ->
	contour.disabled = true
	if oldBounds\equal data\getLineBounds!
	if contour.parent.class == ASS.Section.Drawing
	stats.contoursDraw += 1
	else stats.contoursClip += 1
	return false
	contour.disabled = false


	-- remove invisible lines
	if res.removeInvisible and oldBounds.w == 0
	stats.bytes += #line.raw + 1
	delCnt += 1
	linesToDelete[delCnt], line.ASS = line
	return


	purgedContourCount = 0
	if res.purgeContoursDraw or res.scale2float
	cb = (section) ->
	-- remove invisible contours from drawings
	if res.purgeContoursDraw
	purgedContourCount = removeInvisibleContoursOpt section, oldBounds
	-- un-scale drawings
	if res.scale2float and section.scale > 1
	section.scale\set 1
	stats.scale2float += 1

	data\callback cb, ASS.Section.Drawing
	if purgedContourCount > 0
	if res.purgeContoursDraw and not res.purgeContoursIgnoreHashMismatch and not data\getLineBounds!\equal oldBounds
	line.text = orgText
	data = ASS\parse line
	stats.contoursDrawSkipped += purgedContourCount
	else
	stats.contoursDraw += purgedContourCount
	oldBounds = data\getLineBounds! if res.purgeContoursIgnoreHashMismatch

	-- pogressively build a table of visible lines by frame
	-- which is required to check mergeability of consecutive identical lines
	if res.combineLines
	line.firstFrame = ffms line.start_time
	line.lastFrame = -1 + ffms line.end_time
	for i = line.firstFrame, line.lastFrame
	lbf = linesByFrame[i]
	if lbf
	lbf[lbf.n+1] = line
	lbf.n += 1
	else linesByFrame[i] = {line, n: 1}

	-- collect lines to combine
	if oldBounds.firstHash and not oldBounds.animated
	ltc = linesToCombine[oldBounds.firstHash]
	if ltc
	ltc[ltc.n+1] = line
	ltc.n += 1
	else linesToCombine[oldBounds.firstHash] = {line, n: 1}

	mergeConsecutiveTagSections = if not res.mergeConsecutive
	false
	elseif #res.mergeConsecutiveExcept == 0
	true
	else
	exceptions = list.makeSet res.mergeConsecutiveExcept
	(sourceSection, targetSection) ->
	predicate = (tag) -> exceptions[tag.__tag.name]
	not table.find(sourceSection.tags, predicate) and not table.find targetSection.tags, predicate
	-- clean tags
	data\cleanTags res.cleanLevel, mergeConsecutiveTagSections, res.tagsToKeep, res.tagSortOrder
	newBounds = data\getLineBounds!

	if res.stripComments
	data\stripComments!
	--data\callback stripComments, ASS.Section.Comment

	if res.filterClips or res.removeJunk
	data\modTags tagNames, (tag) ->
	-- remove junk
	if tag.class == ASS.Tag.Unknown
	stats.junk += 1
	return false

	if tag.class == ASS.Tag.ClipVect
	-- un-scale clips
	if res.scale2float and tag.scale>1
	tag.scale\set 1
	stats.scale2float += 1
	-- purge ineffective contours from clips
	if res.purgeContoursClip
	tag\callback removeInvisibleContours

	-- filter clips
	tag.disabled = true
	if data\getLineBounds!\equal newBounds
	stats.clips += 1
	return false
	tag.disabled = false

	data\commit nil, res.cleanLevel == 0
	-- reclaim some memory
	line.ASS, line.undoText = nil

	if orgText != line.text
	if not newBounds\equal oldBounds
	debugError = true
	logger\warn "Cleaning affected output on line #%d, rolling back...", line.humanizedNumber
	logger\warn "—— Before: %s\n—— Parsed: %s\n—— After: %s\n—— Style: %s\n—— Tags: %s", orgText, orgTextParsed, line.text, line.styleRef.name, table.concat orgTagTypes, "; "
	logger\warn "—— Hash Before: %s (%s); Hash After: %s (%s)\n",
	oldBounds.firstHash, oldBounds.animated and "animated" or "static",
	newBounds.firstHash, newBounds.animated and "animated" or "static"
	line.text = orgText
	elseif #line.text < #orgText
	stats.cleaned += 1
	stats.bytes += #orgText - #line.text

	aegisub.cancel! if aegisub.progress.is_cancelled!
	lines\runCallback callback, true

	-- sort lines which are to be combined by time
	sortFunc = (a, b) ->
	return true if a.start_time < b.start_time
	return false if a.start_time > b.start_time
	return true if a.layer < b.layer
	return false if a.layer > b.layer
	return true if a.number < b.number
	return false

	linesToCombineSorted, l = {}, 1
	for _, group in pairs linesToCombine
	continue if group.n < 2
	sort group, sortFunc
	linesToCombineSorted[l] = group
	l += 1
	sort linesToCombineSorted, (a, b) -> sortFunc a[1], b[1]

	-- combine lines
	for group in *linesToCombineSorted
	for j=1, group.n
	linesToDelete[delCnt+cmbCnt+1], cmbCnt, stats.bytes = mergeLines group, j, cmbCnt, stats.bytes, linesByFrame

	lines\replaceLines!
	lines\deleteLines linesToDelete

	logger\warn json.encode {Styles: lines.styles, Configuration: res} if debugError
	logger\warn reportMsg, lineCnt, os.time!-stats.start, stats.cleaned, 100*stats.cleaned/lineCnt,
	delCnt, 100delCnt/lineCnt, cmbCnt, 100cmbCnt/lineCnt, stats.clips, stats.junk,
	stats.contoursClip+stats.contoursDraw, stats.contoursDraw, stats.contoursClip, stats.contoursDrawSkipped,
	stats.scale2float, stats.extra, stats.bytes/1000

	if debugError
	logger\warn [[However, ASSWipe possibly encountered bugs while cleaning.
	Affected lines have been rolled back to their previous state, so your script is most likely fine.
	Please copy the whole log window contents and send them to line0.]]


	return lines\getSelection!


	showDialog = (sub, sel, res) ->
	dlg = {
	main: {
	removeInvisible: class: "checkbox", x: 0, y: 0, width: 2, height: 1, value: true, config: true, label: "Remove invisible lines", hint: hints.removeInvisible
	combineLines: class: "checkbox", x: 0, y: 1, width: 2, height: 1, value: true, config: true, label: "Combine consecutive identical lines", hint: hints.combineLines
	mergeConsecutive: class: "checkbox", x: 2, y: 0, width: 12, height: 1, value: true, config: true, label: "Merge consecutive tag sections unless it contains any of:", hint: hints.mergeConsecutive
	mergeConsecutiveExcept: class: "textbox", x: 2, y: 1, width: 12, height: 2, value: karaokeTags, config: true, hint: hints.mergeConsecutiveExcept

	cleanLevelLabel: class: "label", x: 0, y: 4, width: 1, height: 1, label: "Tag cleanup level: "
	cleanLevel: class: "intedit", x: 1, y: 4, width: 1, height: 1, min: 0, max: 4, value: 4, config: true, hint: hints.cleanLevel
	tagsToKeepLabel: class: "label", x: 4, y: 4, width: 1, height: 1, label: "Keep default tags: "
	tagsToKeep: class: "textbox", x: 4, y: 5, width: 10, height: 2, value: "\\pos", config:true, hint: hints.tagsToKeep
	tagSortOrderLabel: class: "label", x: 4, y: 7, width: 1, height: 1, label: "Tag sort order: "
	stripComments: class: "checkbox", x: 0, y: 5, width: 2, height: 1, value: true, config: true, label: "Strip comments", hint: hints.stripComments
	removeJunk: class: "checkbox", x: 0, y: 6, width: 2, height: 1, value: true, config: true, label: "Remove junk from tag sections", hint: hints.removeJunk
	removeUnknown: class: "checkbox", x: 0, y: 7, width: 2, height: 1, value: true, config: true, label: "Remove unknown tags from tag sections", hint: hints.removeJunk
	tagSortOrder: class: "textbox", x: 4, y: 8, width: 10, height: 3, value: defaultSortOrder, config: true, hint: hints.tagSortOrder

	filterClips: class: "checkbox", x: 0, y: 11, width: 2, height: 1, value: true, config: true, label: "Filter clips", hint: hints.filterClips
	scale2float: class: "checkbox", x: 0, y: 12, width: 2, height: 1, value: true, config: true, label: "Un-scale drawings and clips", hint: hints.scale2float
	fixDrawings: class: "checkbox", x: 0, y: 13, width: 2, height: 1, value: false, config: true, label: "Try to fix broken drawings", hint: hints.fixDrawings
	purgeContoursLabel: class: "label", x: 0, y: 14, width: 2, height: 1, label: "Purge invisible contours: "
	purgeContoursDraw: class: "checkbox", x: 4, y: 14, width: 3, height: 1, value: false, config: true, label: "from drawings", hint: hints.purgeContoursDraw
	purgeContoursClip: class: "checkbox", x: 7, y: 14, width: 6, height: 1, value: false, config: true, label: "from clips", hint: hints.purgeContoursClip
	purgeContoursIgnoreHashMismatch: class: "checkbox", x: 4, y: 15, width: 9, height: 1, value: false, config: true, label: "ignore rendering inconsistencies", hint: hints.purgeContoursIgnoreHashMismatch
	extraDataLabel: class: "label", x: 0, y: 17, width: 1, height: 1, label: "Filter extra data: "
	extraDataMode: class: "dropdown", x: 1, y: 17, width: 1, height: 1, value: "Keep All", config: true, items: {"Keep all", "Remove all", "Keep all except", "Remove all except"}, hint: hints.extraData
	extraDataFilter: class: "textbox", x: 4, y: 17, width: 10, height: 3, value: "", config: true, hint: hints.extraData
	quirksModeLabel: class: "label", x: 0, y: 21, width: 2, height: 1, label: "Quirks mode: "
	quirksMode: class: "dropdown", x: 4, y: 21, width: 2, height: 1, value: "VSFilter", config: true, items: [k for k, v in pairs ASS.Quirks], hint: hints.quirksMode
	}
	}
	options = ConfigHandler dlg, version.configFile, false, script_version, version.configDir
	options\read!
	options\updateInterface "main"
	btn, res = aegisub.dialog.display dlg.main
	if btn
	options\updateConfiguration res, "main"
	options\write!
	ASS.config.quirks = ASS.Quirks[res.quirksMode]
	process sub, sel, res

	version\registerMacro showDialog, ->
	if aegisub.project_properties!.video_file == ""
	return false, "A video must be loaded to run #{script_name}."
	else return true, script_description
No results found