songsenand
/
cgal
mirror of https://github.com/CGAL/cgal
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5279 from sgiraudot/RANSAC-Fix_ptrdiff_overflow-GF 

[RANSAC] Fix ptrdiff overflow
This commit is contained in:
Laurent Rineau 2020-12-16 17:47:45 +01:00
parent 5690767631 a7b2ed13f4
commit fe34a5aa04
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Shape_detection/include/CGAL/Shape_detection/Efficient_RANSAC/Shape_base.h
View File

@ -623,10 +623,11 @@ namespace CGAL {
const std::ptrdiff_t n,
FT &low,
FT &high) {
const FT q = FT(x * n * double(UN - x) * (UN - n) / (UN - 1));
const FT xn = FT(double(x) * double(n));
const FT q = FT(xn * double(UN - x) * (UN - n) / (UN - 1));
const FT sq = CGAL::sqrt(q);
low = (x * n - sq) / UN;
high = (x * n + sq)/UN;
low = (xn - sq) / UN;
high = (xn + sq)/UN;
if (!is_finite<FT>(low) || !is_finite<FT>(high)) {
low = high = 0;