Обработка массива текстов

 @D:\parser\PipeDemo.exe ^
 --input-file="D:\parser\novost.txt" ^
 --input-encoding="utf-8" ^
 --rules-file="D:\parser\rules.xml" ^
 --mssqlrep-connectionstring="Data Source=SYNTAX.RRCKI.RU;Initial Catalog=Scenarios;User Id=sa;Password=REPLACE_PASSWORD" ^
 --pgsink-connection="Host=syntax.rrcki.ru;Port=5433;Username=postgres;Password=REPLACE_PASSWORD;Database=emotional" ^
 --morphmatchregex_pattern="C:\parser\patterns.xml"^
 --guesserweb-host="http://syntax.rrcki.ru:5000"^
 --guesserweb-ntop 10

Emotional Agent Pipeline Parser.
Started              : 2018-09-24 16:37:18.09
Initialized          : 2018-09-24 16:37:19.91
Finished             : 2018-09-24 16:37:40.94

SELECT 'sentences' base, MAX(id) maxid FROM sentences 
UNION ALL
SELECT 'syntax' base, MAX(id) maxid FROM syntax
UNION ALL
SELECT 'facts' base, MAX(id) maxid FROM facts

-- что происходит?
-- по каждой таблице выводит последний процесс, id последнего элемента и хэш
-- при последовательных запусках позволяет следит за измнениями в базе, проверять, какие базы меняет последний процесс

(SELECT 'sentences' base, substring(meta -> 'metadata' -> 'splitter_processor' ->> 'process_id' FOR 8) process, id, substring(hash::text for 5) hash
    , 'matcher: "' || (meta -> 'metadata' -> 'morphology_matcher' ->> 'version') || 
    '", guesser: "' || (meta -> 'metadata' -> 'morphology_guesser' ->> 'version') || 
    '", splitter: "' || (meta -> 'metadata' -> 'splitter_processor' ->> 'version') || 
    '", processor: "' || (meta -> 'metadata' -> 'morphology_processor' ->> 'version') || '"' ver
    , meta -> 'metadata' -> 'splitter' ->> 'text' sent
    , jsonb_pretty(sentences.sentence) cont
    FROM sentences 
ORDER BY id DESC LIMIT 1) 

UNION ALL
(SELECT 'syntax' base, substring(meta -> 'metadata' -> 'splitter_processor' ->> 'process_id' FOR 8) process, id, substring(hash::text for 5) hash
    , 'syntax: "' || (meta -> 'metadata' -> 'syntax_processor' ->> 'version') || 
    '", weighting: "' || (meta -> 'metadata' -> 'weighting_processor' ->> 'version') || '"' ver
    , meta -> 'metadata' -> 'splitter' ->> 'text' sent 
    , jsonb_pretty(syntax.tree) cont
    FROM syntax
ORDER BY id DESC LIMIT 1) 

UNION ALL
(SELECT 'facts' base, substring(meta -> 'metadata' -> 'splitter_processor' ->> 'process_id' FOR 8) process, id, substring(hash::text for 5) hash
    , 'syntax: "' || (meta -> 'metadata' -> 'syntax_processor' ->> 'version') || 
    '", weighting: "' || (meta -> 'metadata' -> 'weighting_processor' ->> 'version') || 
    '", semantics: "' || (meta -> 'metadata' -> 'semantics_processor' ->> 'version') || '"' ver
    , F.meta -> 'metadata' -> 'splitter' ->> 'text' sent
    , '' cont
    FROM facts F
ORDER BY id DESC LIMIT 1)

UNION ALL
(SELECT 'fact_valencies' base, substring(meta -> 'metadata' -> 'splitter_processor' ->> 'process_id' FOR 8) process, FV.id, substring(hash::text for 5) hash
    , '' ver
    , F.meta -> 'metadata' -> 'splitter' ->> 'text' sent
    , '' cont
    FROM fact_valencies FV
LEFT JOIN facts F ON FV.fact = F.id 
ORDER BY FV.id DESC LIMIT 1)

UNION ALL
(SELECT 'referents' base, substring(meta -> 'metadata' -> 'splitter_processor' ->> 'process_id' FOR 8) process, R.id, substring(hash::text for 5) hash
    , '' ver
    , F.meta -> 'metadata' -> 'splitter' ->> 'text' sent
    , '' cont    
    FROM referents R
LEFT JOIN fact_valencies FV ON R.id = FV.referent
LEFT JOIN facts F ON FV.fact = F.id 
ORDER BY R.id DESC LIMIT 1)

UNION ALL
(SELECT 'fact_markers' base, substring(meta -> 'metadata' -> 'splitter_processor' ->> 'process_id' FOR 8) process, FM.id, substring(hash::text for 5) hash
    , '' ver
    , F.meta -> 'metadata' -> 'splitter' ->> 'text' sent
    , '' cont 
    FROM fact_markers FM
LEFT JOIN fact_valencies FV ON FM.valency = FV.id
LEFT JOIN facts F ON FV.fact = F.id 
ORDER BY FM.id DESC LIMIT 1)

SELECT N1.*, B.unknown_token FROM
    (
        SELECT     
           substring(A.procid FOR 8) -- преобразуем json в строку и обрезаем
           , Se.id sentID
           , Se.meta ->'metadata'->'splitter'->>'text' sent
           , COUNT(Sy.id) treeCNT
           , COUNT(F.id) fCNT
           -- проверяем, что для предложения сохранено дерево или факт, значит, предложение разобрано
           , CASE (COUNT(Sy.id) > 0) OR (COUNT(F.id) > 0) WHEN True THEN 1 ELSE NULL END oksent

           FROM 
           (    SELECT meta -> 'metadata' -> 'splitter_processor' ->> 'process_id' procid, MAX(id) maxid
                FROM sentences
                GROUP BY procid
                ORDER BY maxid DESC
                LIMIT 1 -- взять последний разбор
           ) A --берем последний procid
           JOIN sentences Se ON A.procid = Se.meta ->'metadata'->'splitter_processor'->>'process_id' 
           LEFT JOIN facts F ON F.meta ->'metadata'->'morphology'->>'id' = Se.meta ->'metadata'->'morphology'->>'id'
           LEFT JOIN syntax Sy ON Sy.meta ->'metadata'->'morphology'->>'id' = Se.meta ->'metadata'->'morphology'->>'id'           

           GROUP BY 
               A.procid
               , Se.id
           ORDER BY --oksent, 
           sentid DESC

       ) N1

      LEFT JOIN

       (
            select X.token->'token'->>'token_text' unknown_token, X.sentID 
                from (
                    select jsonb_array_elements(sentence -> 'tokens') token, id sentID from sentences
                ) X
            where X.token->'token'->>'token_type' = 'Unknown'
       ) B ON B.sentID = N1.sentID
ORDER BY N1.sentID DESC