Checkout pdf_processing_xcp_legacy.xml in the /Media Server/System Profile/ via Documentum Administrator
Make the following changes under storyboard_pdfstoryboard section.
Update doc_token_width value to 1500
Update doc_token_height value to 1500
Update doc_token_dpi value to 296
Change doc_token_img_quality value from regular to hires

<InnerProfile path="/System/Media Server/System Profiles/storyboard_pdfstoryboard" waitOnCompletion="true" useTargetFormat="true">
<InnerTokenMapping LocalProfileToken="jpeg_lres" InnerProfileToken="doc_token_targetFormat" Literal="true"/>
<InnerTokenMapping Literal="true" InnerProfileToken="add_rendition_properties" LocalProfileToken="false"/>
<InnerTokenMapping LocalProfileToken="1500" InnerProfileToken="doc_token_width" Literal="true"/>
<InnerTokenMapping LocalProfileToken="1500" InnerProfileToken="doc_token_height" Literal="true"/>
<InnerTokenMapping LocalProfileToken="296" InnerProfileToken="doc_token_dpi" Literal="true"/>
<InnerTokenMapping LocalProfileToken="-1" InnerProfileToken="doc_token_pageNumber" Literal="true"/>
<InnerTokenMapping LocalProfileToken="true" InnerProfileToken="enable_activepreview" Literal="true"/>
<InnerTokenMapping LocalProfileToken="true" InnerProfileToken="zip_output" Literal="true"/>
<InnerTokenMapping Literal="true" InnerProfileToken="doc_token_colorSpace" LocalProfileToken="RGB"/>
<InnerTokenMapping Literal="true" InnerProfileToken="doc_token_colorProfile" LocalProfileToken="_use_default"/>
<InnerTokenMapping LocalProfileToken="hires" InnerProfileToken="doc_token_img_quality" Literal="true"/>
<InnerTokenMapping LocalProfileToken="10" InnerProfileToken="doc_token_chunksize" Literal="true"/>
</InnerProfile>

• Checkin the xml back to the same folder path

3. In the storyboard_pdfstoryboard (system profiles), you need to change to load PDFStoryboard first and then followed by Image3 (by default Image3 plugin load first).

<CommandFilePath mptype="PDFStoryboard">
/System/Media Server/Command Line Files/storyboard_pdfstoryboard.xml
</CommandFilePath>
<CommandFilePath mptype="IMAGE3">
/System/Media Server/Command Line Files/storyboard_pdf_imw.xml
</CommandFilePath>

• Checkin the xml profile and restart the Content Transformation services for the changes to take effect.